diff --git a/go-controller/go.mod b/go-controller/go.mod index 062f2ee952..a9513363cc 100644 --- a/go-controller/go.mod +++ b/go-controller/go.mod @@ -35,6 +35,7 @@ require ( github.com/urfave/cli/v2 v2.2.0 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 golang.org/x/sys v0.0.0-20210112080510-489259a85091 + golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gopkg.in/fsnotify/fsnotify.v1 v1.4.7 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/natefinch/lumberjack.v2 v2.0.0 diff --git a/go-controller/pkg/ovn/OCP_HACKS.go b/go-controller/pkg/ovn/OCP_HACKS.go deleted file mode 100644 index ddc34b7005..0000000000 --- a/go-controller/pkg/ovn/OCP_HACKS.go +++ /dev/null @@ -1,91 +0,0 @@ -package ovn - -import ( - "fmt" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - kapi "k8s.io/api/core/v1" - "strings" -) - -// createNodePortLoadBalancers is just a copy of the current node balancer code that will not be invoked during -// local gateway mode + gateway-interface of "none". So we need to still create them ourselves on for the node -// switches (since they will not be created on the GR) -// See https://github.com/openshift/ovn-kubernetes/pull/281 -func createNodePortLoadBalancers(gatewayRouter, nodeName string, sctpSupport bool) error { - // Create 3 load-balancers for north-south traffic for each gateway - // router: UDP, TCP, SCTP - k8sNSLbTCP, k8sNSLbUDP, k8sNSLbSCTP, err := getGatewayLoadBalancers(gatewayRouter) - if err != nil { - return err - } - protoLBMap := map[kapi.Protocol]string{ - kapi.ProtocolTCP: k8sNSLbTCP, - kapi.ProtocolUDP: k8sNSLbUDP, - kapi.ProtocolSCTP: k8sNSLbSCTP, - } - enabledProtos := []kapi.Protocol{kapi.ProtocolTCP, kapi.ProtocolUDP} - if sctpSupport { - enabledProtos = append(enabledProtos, kapi.ProtocolSCTP) - } - var stdout, stderr string - for _, proto := range enabledProtos { - if protoLBMap[proto] == "" { - protoLBMap[proto], stderr, err = util.RunOVNNbctl("--", "create", - "load_balancer", - fmt.Sprintf("external_ids:%s_lb_gateway_router=%s", proto, gatewayRouter), - fmt.Sprintf("protocol=%s", strings.ToLower(string(proto)))) - if err != nil { - return fmt.Errorf("failed to create load balancer for gateway router %s for protocol %s: "+ - "stderr: %q, error: %v", gatewayRouter, proto, stderr, err) - } - } - } - - // Local gateway mode does not use GR for ingress node port traffic, it uses mp0 instead - if config.Gateway.Mode != config.GatewayModeLocal { - // Add north-south load-balancers to the gateway router. - lbString := fmt.Sprintf("%s,%s", protoLBMap[kapi.ProtocolTCP], protoLBMap[kapi.ProtocolUDP]) - if sctpSupport { - lbString = lbString + "," + protoLBMap[kapi.ProtocolSCTP] - } - stdout, stderr, err = util.RunOVNNbctl("set", "logical_router", gatewayRouter, "load_balancer="+lbString) - if err != nil { - return fmt.Errorf("failed to set north-south load-balancers to the "+ - "gateway router %s, stdout: %q, stderr: %q, error: %v", - gatewayRouter, stdout, stderr, err) - } - } - // Also add north-south load-balancers to local switches for pod -> nodePort traffic - stdout, stderr, err = util.RunOVNNbctl("get", "logical_switch", nodeName, "load_balancer") - if err != nil { - return fmt.Errorf("failed to get load-balancers on the node switch %s, stdout: %q, "+ - "stderr: %q, error: %v", nodeName, stdout, stderr, err) - } - for _, proto := range enabledProtos { - if !strings.Contains(stdout, protoLBMap[proto]) { - stdout, stderr, err = util.RunOVNNbctl("ls-lb-add", nodeName, protoLBMap[proto]) - if err != nil { - return fmt.Errorf("failed to add north-south load-balancer %s to the "+ - "node switch %s, stdout: %q, stderr: %q, error: %v", - protoLBMap[proto], nodeName, stdout, stderr, err) - } - } - } - return nil -} - -// gatewayInitMinimal sets up the minimal configuration needed for N/S traffic to work in Local gateway mode. In this -// case we do not need any GR or join switch, just nodePort load balancers on the node switch -// See https://github.com/openshift/ovn-kubernetes/pull/281 -func gatewayInitMinimal(nodeName string, l3GatewayConfig *util.L3GatewayConfig, sctpSupport bool) error { - gatewayRouter := types.GWRouterPrefix + nodeName - if l3GatewayConfig.NodePortEnable { - err := createNodePortLoadBalancers(gatewayRouter, nodeName, sctpSupport) - if err != nil { - return err - } - } - return nil -} diff --git a/go-controller/pkg/ovn/controller/services/load_balancer.go b/go-controller/pkg/ovn/controller/services/load_balancer.go new file mode 100644 index 0000000000..15b7e8e6bd --- /dev/null +++ b/go-controller/pkg/ovn/controller/services/load_balancer.go @@ -0,0 +1,475 @@ +package services + +import ( + "fmt" + "reflect" + "strings" + + globalconfig "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + ovnlb "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + + v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1beta1" + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" +) + +// magic string used in vips to indicate that the node's physical +// ips should be substituted in +const placeholderNodeIPs = "node" + +// lbConfig is the abstract desired load balancer configuration. +// vips and endpoints are mixed families. +type lbConfig struct { + vips []string // just ip or the special value "node" for the node's physical IPs (i.e. NodePort) + protocol v1.Protocol // TCP, UDP, or SCTP + inport int32 // the incoming (virtual) port number + eps util.LbEndpoints + + // if true, then vips added on the router are in "local" mode + // that means, skipSNAT, and remove any non-local endpoints. + // (see below) + externalTrafficLocal bool +} + +// just used for consistent ordering +var protos = []v1.Protocol{ + v1.ProtocolTCP, + v1.ProtocolUDP, + v1.ProtocolSCTP, +} + +// buildServiceLBConfigs generates the abstract load balancer(s) configurations for each service. The abstract configurations +// are then expanded in buildClusterLBs and buildPerNodeLBs to the full list of OVN LBs desired. +// +// It creates two lists of configurations: +// - the per-node configs, which are load balancers that, for some reason must be expanded per-node. (see below for why) +// - the cluster-wide configs, which are load balancers that can be the same across the whole cluster. +// +// For a "standard" ClusterIP service (possibly with ExternalIPS or external LoadBalancer Status IPs), +// a single cluster-wide LB will be created. +// +// Per-node LBs will be created for +// - services with NodePort set +// - services with host-network endpoints (for shared gateway mode) +// - services with ExternalTrafficPolicy=Local +func buildServiceLBConfigs(service *v1.Service, endpointSlices []*discovery.EndpointSlice) (perNodeConfigs []lbConfig, clusterConfigs []lbConfig) { + // For each svcPort, determine if it will be applied per-node or cluster-wide + for _, svcPort := range service.Spec.Ports { + eps := util.GetLbEndpoints(endpointSlices, svcPort) + + // if ExternalTrafficPolicy is local, then we need to do things a bit differently + externalTrafficLocal := globalconfig.Gateway.Mode == globalconfig.GatewayModeShared && + service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal + + // NodePort services get a per-node load balancer, but with the node's physical IP as the vip + // Thus, the vip "node" will be expanded later. + if svcPort.NodePort != 0 { + nodePortLBConfig := lbConfig{ + protocol: svcPort.Protocol, + inport: svcPort.NodePort, + vips: []string{placeholderNodeIPs}, // shortcut for all-physical-ips + eps: eps, + externalTrafficLocal: externalTrafficLocal, + } + perNodeConfigs = append(perNodeConfigs, nodePortLBConfig) + } + + // Build up list of vips + vips := append([]string{}, service.Spec.ClusterIPs...) + // Handle old clusters w/o v6 support + if len(vips) == 0 { + vips = []string{service.Spec.ClusterIP} + } + externalVips := []string{} + // ExternalIP + externalVips = append(externalVips, service.Spec.ExternalIPs...) + // LoadBalancer status + for _, ingress := range service.Status.LoadBalancer.Ingress { + if ingress.IP != "" { + externalVips = append(externalVips, ingress.IP) + } + } + + // if ETP=Local, then treat ExternalIPs and LoadBalancer IPs specially + // otherwise, they're just cluster IPs + if externalTrafficLocal && len(externalVips) > 0 { + externalIPConfig := lbConfig{ + protocol: svcPort.Protocol, + inport: svcPort.Port, + vips: externalVips, + eps: eps, + externalTrafficLocal: true, + } + perNodeConfigs = append(perNodeConfigs, externalIPConfig) + } else { + vips = append(vips, externalVips...) + } + + // Build the clusterIP config + // This is NEVER influenced by ExternalTrafficPolicy + clusterIPConfig := lbConfig{ + protocol: svcPort.Protocol, + inport: svcPort.Port, + vips: vips, + eps: eps, + externalTrafficLocal: false, // always false for ClusterIPs + } + + // Normally, the ClusterIP LB is global (on all node switches and routers), + // unless both of the following are true: + // - We're in shared gateway mode, and + // - Any of the endpoints are host-network + // + // In that case, we need to create per-node LBs. + if globalconfig.Gateway.Mode == globalconfig.GatewayModeShared && + (hasHostEndpoints(eps.V4IPs) || hasHostEndpoints(eps.V6IPs)) { + perNodeConfigs = append(perNodeConfigs, clusterIPConfig) + } else { + clusterConfigs = append(clusterConfigs, clusterIPConfig) + } + } + + return +} + +// makeLBName creates the load balancer name - used to minimize churn +func makeLBName(service *v1.Service, proto v1.Protocol, scope string) string { + return fmt.Sprintf("Service_%s/%s_%s_%s", + service.Namespace, service.Name, + proto, scope, + ) +} + +// buildClusterLBs takes a list of lbConfigs and aggregates them +// in to one ovn LB per protocol. +// +// It takes a list of (proto:[vips]:port -> [endpoints]) configs and re-aggregates +// them to a list of (proto:[vip:port -> [endpoint:port]]) +// This load balancer is attached to all node switches. In shared-GW mode, it is also on all routers +func buildClusterLBs(service *v1.Service, configs []lbConfig, nodeInfos []nodeInfo) []ovnlb.LB { + nodeSwitches := make([]string, 0, len(nodeInfos)) + nodeRouters := make([]string, 0, len(nodeInfos)) + for _, node := range nodeInfos { + nodeSwitches = append(nodeSwitches, node.switchName) + // For shared gateway, add to the node's GWR as well. + // The node may not have a gateway router - it might be waiting initialization, or + // might have disabled GWR creation via the k8s.ovn.org/l3-gateway-config annotation + if globalconfig.Gateway.Mode == globalconfig.GatewayModeShared && node.gatewayRouterName != "" { + nodeRouters = append(nodeRouters, node.gatewayRouterName) + } + } + + cbp := configsByProto(configs) + + out := []ovnlb.LB{} + for _, proto := range protos { + cfgs, ok := cbp[proto] + if !ok { + continue + } + lb := ovnlb.LB{ + Name: makeLBName(service, proto, "cluster"), + Protocol: string(proto), + ExternalIDs: util.ExternalIDsForObject(service), + Opts: lbOpts(service), + + Switches: nodeSwitches, + Routers: nodeRouters, + } + + for _, config := range cfgs { + if config.externalTrafficLocal { + klog.Errorf("BUG: service %s/%s has routerLocalMode=true for cluster-wide lbConfig", + service.Namespace, service.Name) + } + + v4targets := make([]ovnlb.Addr, 0, len(config.eps.V4IPs)) + for _, tgt := range config.eps.V4IPs { + v4targets = append(v4targets, ovnlb.Addr{ + IP: tgt, + Port: config.eps.Port, + }) + } + + v6targets := make([]ovnlb.Addr, 0, len(config.eps.V6IPs)) + for _, tgt := range config.eps.V6IPs { + v6targets = append(v6targets, ovnlb.Addr{ + IP: tgt, + Port: config.eps.Port, + }) + } + + rules := make([]ovnlb.LBRule, 0, len(config.vips)) + for _, vip := range config.vips { + if vip == placeholderNodeIPs { + klog.Errorf("BUG: service %s/%s has a \"node\" vip for a cluster-wide lbConfig", + service.Namespace, service.Name) + continue + } + targets := v4targets + if utilnet.IsIPv6String(vip) { + targets = v6targets + } + + rules = append(rules, ovnlb.LBRule{ + Source: ovnlb.Addr{ + IP: vip, + Port: config.inport, + }, + Targets: targets, + }) + } + lb.Rules = append(lb.Rules, rules...) + } + + out = append(out, lb) + } + return out +} + +// buildPerNodeLBs takes a list of lbConfigs and expands them to one LB per protocol per node +// This works differently based on whether or not we're in shared or local gateway mode. +// +// For local gateway, per-node lbs are created for: +// - nodePort services, attached to each node's switch, vips are node's physical IPs +// +// For shared gateway, per-node lbs are created for +// - clusterip services with host-network endpoints are attached to each node's gateway router + switch +// - nodeport services are attached to each node's gateway router + switch, vips are node's physical IPs +// - any services with host-network endpoints +// - services with external IPs / LoadBalancer Status IPs +// +// HOWEVER, we need to replace, on each nodes gateway router only, any host-network endpoints with a special loopback address +// see https://github.com/ovn-org/ovn-kubernetes/blob/master/docs/design/host_to_services_OpenFlow.md +// This is for host -> serviceip -> host hairpin +// +// For ExternalTrafficPolicy, all "External" IPs (NodePort, ExternalIPs, Loadbalancer Status) have: +// - targets filtered to only local targets +// - SkipSNAT enabled +// This results in the creation of an additional load balancer on the GatewayRouters. +func buildPerNodeLBs(service *v1.Service, configs []lbConfig, nodes []nodeInfo) []ovnlb.LB { + cbp := configsByProto(configs) + eids := util.ExternalIDsForObject(service) + + out := make([]ovnlb.LB, 0, len(nodes)*len(configs)) + + // output is one LB per node per protocol + // with one rule per vip + for _, node := range nodes { + for _, proto := range protos { + configs, ok := cbp[proto] + if !ok { + continue + } + + // local gateway mode - attach to switch only + // shared gateway mode - attach to router & switch, + // rules may or may not be different + // localRouterRules are rules with no snat + routerRules := make([]ovnlb.LBRule, 0, len(configs)) + noSNATRouterRules := make([]ovnlb.LBRule, 0) + switchRules := make([]ovnlb.LBRule, 0, len(configs)) + + for _, config := range configs { + vips := config.vips + + routerV4targetips := config.eps.V4IPs + routerV6targetips := config.eps.V6IPs + + // shared gateway needs to "massage" some of the targets + if globalconfig.Gateway.Mode == "shared" { + // for ExternalTrafficPolicy=Local, then remove non-local endpoints from the router targets + if config.externalTrafficLocal { + routerV4targetips = util.FilterIPsSlice(routerV4targetips, node.nodeSubnets(), true) + routerV6targetips = util.FilterIPsSlice(routerV6targetips, node.nodeSubnets(), true) + } + + // at this point, the targets may be empty + + // any targets local to the node need to have a special + // harpin IP added, but only for the router LB + routerV4targetips = util.UpdateIPsSlice(routerV4targetips, node.nodeIPs, []string{types.V4HostMasqueradeIP}) + routerV6targetips = util.UpdateIPsSlice(routerV6targetips, node.nodeIPs, []string{types.V6HostMasqueradeIP}) + } + + routerV4targets := ovnlb.JoinHostsPort(routerV4targetips, config.eps.Port) + routerV6targets := ovnlb.JoinHostsPort(routerV6targetips, config.eps.Port) + + switchV4Targets := ovnlb.JoinHostsPort(config.eps.V4IPs, config.eps.Port) + switchV6Targets := ovnlb.JoinHostsPort(config.eps.V6IPs, config.eps.Port) + + // Substitute the special vip "node" for the node's physical ips + // This is used for nodeport + vips = make([]string, 0, len(vips)) + for _, vip := range config.vips { + if vip == placeholderNodeIPs { + vips = append(vips, node.nodeIPs...) + } else { + vips = append(vips, vip) + } + } + + for _, vip := range vips { + isv6 := utilnet.IsIPv6String((vip)) + // build switch rules + targets := switchV4Targets + if isv6 { + targets = switchV6Targets + } + + switchRules = append(switchRules, ovnlb.LBRule{ + Source: ovnlb.Addr{IP: vip, Port: config.inport}, + Targets: targets, + }) + + // For shared gateway, there is also a per-router rule + // with targets that *may* be different + if globalconfig.Gateway.Mode == "shared" { + targets := routerV4targets + if isv6 { + targets = routerV6targets + } + rule := ovnlb.LBRule{ + Source: ovnlb.Addr{IP: vip, Port: config.inport}, + Targets: targets, + } + + // in other words, is this ExternalTrafficPolicy=local? + // if so, this gets a separate load balancer with SNAT disabled + // (but there's no need to do this if the list of targets is empty) + if config.externalTrafficLocal && len(targets) > 0 { + noSNATRouterRules = append(noSNATRouterRules, rule) + } else { + routerRules = append(routerRules, rule) + } + } + } + } + + // If switch and router rules are identical, coalesce + if reflect.DeepEqual(switchRules, routerRules) && len(switchRules) > 0 && node.gatewayRouterName != "" { + out = append(out, ovnlb.LB{ + Name: makeLBName(service, proto, "node_router+switch_"+node.name), + Protocol: string(proto), + ExternalIDs: eids, + Opts: lbOpts(service), + Routers: []string{node.gatewayRouterName}, + Switches: []string{node.switchName}, + Rules: routerRules, + }) + } else { + if len(routerRules) > 0 && node.gatewayRouterName != "" { + out = append(out, ovnlb.LB{ + Name: makeLBName(service, proto, "node_router_"+node.name), + Protocol: string(proto), + ExternalIDs: eids, + Opts: lbOpts(service), + Routers: []string{node.gatewayRouterName}, + Rules: routerRules, + }) + } + if len(noSNATRouterRules) > 0 && node.gatewayRouterName != "" { + lb := ovnlb.LB{ + Name: makeLBName(service, proto, "node_local_router_"+node.name), + Protocol: string(proto), + ExternalIDs: eids, + Opts: lbOpts(service), + Routers: []string{node.gatewayRouterName}, + Rules: noSNATRouterRules, + } + lb.Opts.SkipSNAT = true + out = append(out, lb) + } + + if len(switchRules) > 0 { + out = append(out, ovnlb.LB{ + Name: makeLBName(service, proto, "node_switch_"+node.name), + Protocol: string(proto), + ExternalIDs: eids, + Opts: lbOpts(service), + Switches: []string{node.switchName}, + Rules: switchRules, + }) + } + } + } + } + + merged := mergeLBs(out) + if len(merged) != len(out) { + klog.V(5).Infof("Service %s/%s merged %d LBs to %d", + service.Namespace, service.Name, + len(out), len(merged)) + } + + return merged +} + +// configsByProto buckets a list of configs by protocol (tcp, udp, sctp) +func configsByProto(configs []lbConfig) map[v1.Protocol][]lbConfig { + out := map[v1.Protocol][]lbConfig{} + for _, config := range configs { + out[config.protocol] = append(out[config.protocol], config) + } + return out +} + +// lbOpts generates the OVN load balancer options from the kubernetes Service. +func lbOpts(service *v1.Service) ovnlb.LBOpts { + return ovnlb.LBOpts{ + Unidling: svcNeedsIdling(service.GetAnnotations()), + Affinity: service.Spec.SessionAffinity == v1.ServiceAffinityClientIP, + SkipSNAT: false, // never service-wide, ExternalTrafficPolicy-specific + } +} + +// mergeLBs joins two LBs together if it is safe to do so. +// +// an LB can be merged if the protocol, rules, and options are the same, +// and only the switches and routers are different. +func mergeLBs(lbs []ovnlb.LB) []ovnlb.LB { + if len(lbs) == 1 { + return lbs + } + out := make([]ovnlb.LB, 0, len(lbs)) + +outer: + for _, lb := range lbs { + for i := range out { + // If mergeable, rather than inserting lb to out, just add switches and routers + // and drop + if canMergeLB(lb, out[i]) { + out[i].Switches = append(out[i].Switches, lb.Switches...) + out[i].Routers = append(out[i].Routers, lb.Routers...) + + if !strings.HasSuffix(out[i].Name, "_merged") { + out[i].Name += "_merged" + } + continue outer + } + } + out = append(out, lb) + } + + return out +} + +// canMergeLB returns true if two LBs are mergeable. +// We know that the ExternalIDs will be the same, so we don't need to compare them. +// All that matters is the protocol and rules are the same. +func canMergeLB(a, b ovnlb.LB) bool { + if a.Protocol != b.Protocol { + return false + } + + if !reflect.DeepEqual(a.Opts, b.Opts) { + return false + } + + // While rules are actually a set, we generate all our lbConfigs from a single source + // so the ordering will be the same. Thus, we can cheat and just reflect.DeepEqual + return reflect.DeepEqual(a.Rules, b.Rules) +} diff --git a/go-controller/pkg/ovn/controller/services/load_balancer_test.go b/go-controller/pkg/ovn/controller/services/load_balancer_test.go new file mode 100644 index 0000000000..a4cdbef826 --- /dev/null +++ b/go-controller/pkg/ovn/controller/services/load_balancer_test.go @@ -0,0 +1,1350 @@ +package services + +import ( + "fmt" + "net" + "testing" + + globalconfig "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + ovnlb "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "github.com/stretchr/testify/assert" + + v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + utilpointer "k8s.io/utils/pointer" +) + +func Test_buildServiceLBConfigs(t *testing.T) { + oldClusterSubnet := globalconfig.Default.ClusterSubnets + oldGwMode := globalconfig.Gateway.Mode + defer func() { + globalconfig.Gateway.Mode = oldGwMode + globalconfig.Default.ClusterSubnets = oldClusterSubnet + }() + _, cidr4, _ := net.ParseCIDR("10.128.0.0/16") + _, cidr6, _ := net.ParseCIDR("fe00::/64") + globalconfig.Default.ClusterSubnets = []globalconfig.CIDRNetworkEntry{{cidr4, 26}, {cidr6, 26}} + + // constants + serviceName := "foo" + ns := "testns" + portName := "port80" + portName1 := "port81" + inport := int32(80) + outport := int32(8080) + inport1 := int32(81) + outport1 := int32(8081) + outportstr := intstr.FromInt(int(outport)) + emptyEPs := util.LbEndpoints{V4IPs: []string{}, V6IPs: []string{}, Port: 0} + tcp := v1.ProtocolTCP + udp := v1.ProtocolUDP + + // make slices + // nil slice = don't use this family + // empty slice = family is empty + makeSlices := func(v4ips, v6ips []string, proto v1.Protocol) []*discovery.EndpointSlice { + out := []*discovery.EndpointSlice{} + if v4ips != nil && len(v4ips) == 0 { + out = append(out, &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName + "ab1", + Namespace: ns, + Labels: map[string]string{discovery.LabelServiceName: serviceName}, + }, + Ports: []discovery.EndpointPort{}, + AddressType: discovery.AddressTypeIPv4, + Endpoints: []discovery.Endpoint{}, + }) + } else if v4ips != nil { + out = append(out, &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName + "ab1", + Namespace: ns, + Labels: map[string]string{discovery.LabelServiceName: serviceName}, + }, + Ports: []discovery.EndpointPort{{ + Protocol: &proto, + Port: &outport, + Name: &portName, + }}, + AddressType: discovery.AddressTypeIPv4, + Endpoints: []discovery.Endpoint{ + { + Conditions: discovery.EndpointConditions{ + Ready: utilpointer.BoolPtr(true), + }, + Addresses: v4ips, + }, + }, + }) + } + + if v6ips != nil && len(v6ips) == 0 { + out = append(out, &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName + "ab2", + Namespace: ns, + Labels: map[string]string{discovery.LabelServiceName: serviceName}, + }, + Ports: []discovery.EndpointPort{}, + AddressType: discovery.AddressTypeIPv6, + Endpoints: []discovery.Endpoint{}, + }) + } else if v6ips != nil { + out = append(out, &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName + "ab2", + Namespace: ns, + Labels: map[string]string{discovery.LabelServiceName: serviceName}, + }, + Ports: []discovery.EndpointPort{{ + Protocol: &proto, + Port: &outport, + Name: &portName, + }}, + AddressType: discovery.AddressTypeIPv6, + Endpoints: []discovery.Endpoint{ + { + Conditions: discovery.EndpointConditions{ + Ready: utilpointer.BoolPtr(true), + }, + Addresses: v6ips, + }, + }, + }) + } + + return out + } + + type args struct { + service *v1.Service + slices []*discovery.EndpointSlice + } + tests := []struct { + name string + args args + + resultSharedGatewayCluster []lbConfig + resultSharedGatewayNode []lbConfig + + resultLocalGatewayNode []lbConfig + resultLocalGatewayCluster []lbConfig + + resultsSame bool //if true, then just use the SharedGateway results for the LGW test + }{ + { + name: "v4 clusterip, one port, no endpoints", + args: args{ + slices: makeSlices([]string{}, nil, v1.ProtocolTCP), + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1"}, + Ports: []v1.ServicePort{{ + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + }}, + }, + }, + }, + resultSharedGatewayCluster: []lbConfig{{ + vips: []string{"192.168.1.1"}, + protocol: v1.ProtocolTCP, + inport: 80, + eps: emptyEPs, + }}, + resultsSame: true, + }, + { + name: "v4 clusterip, one port, endpoints", + args: args{ + slices: makeSlices([]string{"10.128.0.2"}, nil, v1.ProtocolTCP), + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1"}, + Ports: []v1.ServicePort{{ + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + }}, + }, + }, + }, + resultSharedGatewayCluster: []lbConfig{{ + vips: []string{"192.168.1.1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2"}, + V6IPs: []string{}, + Port: outport, + }, + }}, + resultsSame: true, + }, + { + name: "v4 clusterip, two tcp ports, endpoints", + args: args{ + slices: []*discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName + "ab1", + Namespace: ns, + Labels: map[string]string{discovery.LabelServiceName: serviceName}, + }, + Ports: []discovery.EndpointPort{ + { + Name: &portName, + Protocol: &tcp, + Port: &outport, + }, { + Name: &portName1, + Protocol: &tcp, + Port: &outport1, + }, + }, + AddressType: discovery.AddressTypeIPv4, + Endpoints: []discovery.Endpoint{ + { + Conditions: discovery.EndpointConditions{ + Ready: utilpointer.BoolPtr(true), + }, + Addresses: []string{"10.128.0.2", "10.128.1.2"}, + }, + }, + }, + }, + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1"}, + Ports: []v1.ServicePort{ + { + Name: portName, + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + }, + { + Name: portName1, + Port: inport1, + Protocol: v1.ProtocolTCP, + TargetPort: intstr.FromInt(int(outport1)), + }, + }, + }, + }, + }, + resultsSame: true, + resultSharedGatewayCluster: []lbConfig{ + { + vips: []string{"192.168.1.1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2", "10.128.1.2"}, + V6IPs: []string{}, + Port: outport, + }, + }, + { + vips: []string{"192.168.1.1"}, + protocol: v1.ProtocolTCP, + inport: inport1, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2", "10.128.1.2"}, + V6IPs: []string{}, + Port: outport1, + }, + }, + }, + }, + { + name: "v4 clusterip, one tcp, one udp port, endpoints", + args: args{ + slices: []*discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName + "ab1", + Namespace: ns, + Labels: map[string]string{discovery.LabelServiceName: serviceName}, + }, + Ports: []discovery.EndpointPort{ + { + Name: &portName, + Protocol: &tcp, + Port: &outport, + }, { + Name: &portName1, + Protocol: &udp, + Port: &outport, + }, + }, + AddressType: discovery.AddressTypeIPv4, + Endpoints: []discovery.Endpoint{ + { + Conditions: discovery.EndpointConditions{ + Ready: utilpointer.BoolPtr(true), + }, + Addresses: []string{"10.128.0.2", "10.128.1.2"}, + }, + }, + }, + }, + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1"}, + Ports: []v1.ServicePort{ + { + Name: portName, + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + }, + { + Name: portName1, + Port: inport, + Protocol: v1.ProtocolUDP, + TargetPort: intstr.FromInt(int(outport)), + }, + }, + }, + }, + }, + resultsSame: true, + resultSharedGatewayCluster: []lbConfig{ + { + vips: []string{"192.168.1.1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2", "10.128.1.2"}, + V6IPs: []string{}, + Port: outport, + }, + }, + { + vips: []string{"192.168.1.1"}, + protocol: v1.ProtocolUDP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2", "10.128.1.2"}, + V6IPs: []string{}, + Port: outport, + }, + }, + }, + }, + { + name: "dual-stack clusterip, one port, endpoints", + args: args{ + slices: makeSlices([]string{"10.128.0.2"}, []string{"fe00::1:1"}, v1.ProtocolTCP), + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1", "2002::1"}, + Ports: []v1.ServicePort{{ + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + }}, + }, + }, + }, + resultsSame: true, + resultSharedGatewayCluster: []lbConfig{{ + vips: []string{"192.168.1.1", "2002::1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2"}, + V6IPs: []string{"fe00::1:1"}, + Port: outport, + }, + }}, + }, + { + name: "dual-stack clusterip, one port, endpoints, external ips + lb status", + args: args{ + slices: makeSlices([]string{"10.128.0.2"}, []string{"fe00::1:1"}, v1.ProtocolTCP), + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1", "2002::1"}, + Ports: []v1.ServicePort{{ + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + }}, + ExternalIPs: []string{"4.2.2.2", "42::42"}, + }, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{{ + IP: "5.5.5.5", + }}, + }, + }, + }, + }, + resultsSame: true, + resultSharedGatewayCluster: []lbConfig{{ + vips: []string{"192.168.1.1", "2002::1", "4.2.2.2", "42::42", "5.5.5.5"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2"}, + V6IPs: []string{"fe00::1:1"}, + Port: outport, + }, + }}, + }, + { + name: "dual-stack clusterip, one port, endpoints, external ips + lb status, ExternalTrafficPolicy", + args: args{ + slices: makeSlices([]string{"10.128.0.2"}, []string{"fe00::1:1"}, v1.ProtocolTCP), + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1", "2002::1"}, + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, + Ports: []v1.ServicePort{{ + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + }}, + ExternalIPs: []string{"4.2.2.2", "42::42"}, + }, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{{ + IP: "5.5.5.5", + }}, + }, + }, + }, + }, + resultsSame: false, + resultSharedGatewayCluster: []lbConfig{ + { + vips: []string{"192.168.1.1", "2002::1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2"}, + V6IPs: []string{"fe00::1:1"}, + Port: outport, + }, + }, + }, + resultSharedGatewayNode: []lbConfig{ + { + vips: []string{"4.2.2.2", "42::42", "5.5.5.5"}, + protocol: v1.ProtocolTCP, + inport: inport, + externalTrafficLocal: true, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2"}, + V6IPs: []string{"fe00::1:1"}, + Port: outport, + }, + }, + }, + + resultLocalGatewayCluster: []lbConfig{ + { + vips: []string{"192.168.1.1", "2002::1", "4.2.2.2", "42::42", "5.5.5.5"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2"}, + V6IPs: []string{"fe00::1:1"}, + Port: outport, + }, + }, + }, + }, + { + name: "dual-stack clusterip, one port, endpoints, nodePort", + args: args{ + slices: makeSlices([]string{"10.128.0.2"}, []string{"fe00::1:1"}, v1.ProtocolTCP), + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1", "2002::1"}, + Ports: []v1.ServicePort{{ + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + NodePort: 5, + }}, + }, + }, + }, + resultsSame: true, + resultSharedGatewayCluster: []lbConfig{{ + vips: []string{"192.168.1.1", "2002::1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2"}, + V6IPs: []string{"fe00::1:1"}, + Port: outport, + }, + }}, + resultSharedGatewayNode: []lbConfig{{ + vips: []string{"node"}, + protocol: v1.ProtocolTCP, + inport: 5, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2"}, + V6IPs: []string{"fe00::1:1"}, + Port: outport, + }, + }}, + }, + { + name: "dual-stack clusterip, one port, endpoints, nodePort, hostNetwork", + args: args{ + // These slices are outside of the config, and thus are host network + slices: makeSlices([]string{"192.168.0.1"}, []string{"2001::1"}, v1.ProtocolTCP), + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1", "2002::1"}, + Ports: []v1.ServicePort{{ + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + NodePort: 5, + }}, + }, + }, + }, + // In shared gateway mode, nodeport and host-network-pods must be per-node + resultSharedGatewayNode: []lbConfig{ + { + vips: []string{"node"}, + protocol: v1.ProtocolTCP, + inport: 5, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"2001::1"}, + Port: outport, + }, + }, + { + vips: []string{"192.168.1.1", "2002::1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"2001::1"}, + Port: outport, + }, + }, + }, + // in local gateway mode, only nodePort is per-node + resultLocalGatewayNode: []lbConfig{ + { + vips: []string{"node"}, + protocol: v1.ProtocolTCP, + inport: 5, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"2001::1"}, + Port: outport, + }, + }, + }, + resultLocalGatewayCluster: []lbConfig{ + { + vips: []string{"192.168.1.1", "2002::1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"2001::1"}, + Port: outport, + }, + }, + }, + }, + { + name: "dual-stack clusterip, one port, endpoints, nodePort, hostNetwork, ExternalTrafficPolicy=Local", + args: args{ + // These slices are outside of the config, and thus are host network + slices: makeSlices([]string{"192.168.0.1"}, []string{"2001::1"}, v1.ProtocolTCP), + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeNodePort, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1", "2002::1"}, + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, + Ports: []v1.ServicePort{{ + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + NodePort: 5, + }}, + }, + }, + }, + // In shared gateway mode, nodeport and host-network-pods must be per-node + resultSharedGatewayNode: []lbConfig{ + { + vips: []string{"node"}, + protocol: v1.ProtocolTCP, + inport: 5, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"2001::1"}, + Port: outport, + }, + externalTrafficLocal: true, + }, + { + vips: []string{"192.168.1.1", "2002::1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"2001::1"}, + Port: outport, + }, + }, + }, + // in local gateway mode, only nodePort is per-node + resultLocalGatewayNode: []lbConfig{ + { + vips: []string{"node"}, + protocol: v1.ProtocolTCP, + inport: 5, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"2001::1"}, + Port: outport, + }, + }, + }, + resultLocalGatewayCluster: []lbConfig{ + { + vips: []string{"192.168.1.1", "2002::1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"2001::1"}, + Port: outport, + }, + }, + }, + }, + { + name: "dual-stack clusterip, one port, endpoints, hostNetwork", + args: args{ + // These slices are outside of the config, and thus are host network + slices: makeSlices([]string{"192.168.0.1"}, []string{"2001::1"}, v1.ProtocolTCP), + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1", "2002::1"}, + Ports: []v1.ServicePort{{ + Port: inport, + Protocol: v1.ProtocolTCP, + TargetPort: outportstr, + }}, + }, + }, + }, + // In shared gateway mode, nodeport and host-network-pods must be per-node + resultSharedGatewayNode: []lbConfig{ + { + vips: []string{"192.168.1.1", "2002::1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"2001::1"}, + Port: outport, + }, + }, + }, + resultLocalGatewayCluster: []lbConfig{ + { + vips: []string{"192.168.1.1", "2002::1"}, + protocol: v1.ProtocolTCP, + inport: inport, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"2001::1"}, + Port: outport, + }, + }, + }, + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprintf("%d_%s", i, tt.name), func(t *testing.T) { + globalconfig.Gateway.Mode = globalconfig.GatewayModeShared + perNode, clusterWide := buildServiceLBConfigs(tt.args.service, tt.args.slices) + assert.EqualValues(t, tt.resultSharedGatewayNode, perNode, "SGW per-node configs should be equal") + assert.EqualValues(t, tt.resultSharedGatewayCluster, clusterWide, "SGW cluster-wide configs should be equal") + + globalconfig.Gateway.Mode = globalconfig.GatewayModeLocal + perNode, clusterWide = buildServiceLBConfigs(tt.args.service, tt.args.slices) + if tt.resultsSame { + assert.EqualValues(t, tt.resultSharedGatewayNode, perNode, "LGW per-node configs should be equal") + assert.EqualValues(t, tt.resultSharedGatewayCluster, clusterWide, "LGW cluster-wide configs should be equal") + } else { + assert.EqualValues(t, tt.resultLocalGatewayNode, perNode, "LGW per-node configs should be equal") + assert.EqualValues(t, tt.resultLocalGatewayCluster, clusterWide, "LGW cluster-wide configs should be equal") + } + }) + } +} + +func Test_buildClusterLBs(t *testing.T) { + name := "foo" + namespace := "testns" + + oldGwMode := globalconfig.Gateway.Mode + defer func() { + globalconfig.Gateway.Mode = oldGwMode + }() + globalconfig.Gateway.Mode = globalconfig.GatewayModeShared + + defaultService := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + }, + } + + defaultNodes := []nodeInfo{ + { + name: "node-a", + nodeIPs: []string{"10.0.0.1"}, + gatewayRouterName: "gr-node-a", + switchName: "switch-node-a", + }, + { + name: "node-b", + nodeIPs: []string{"10.0.0.2"}, + gatewayRouterName: "gr-node-b", + switchName: "switch-node-b", + }, + } + + defaultExternalIDs := map[string]string{ + "k8s.ovn.org/kind": "Service", + "k8s.ovn.org/owner": fmt.Sprintf("%s/%s", namespace, name), + } + + defaultRouters := []string{"gr-node-a", "gr-node-b"} + defaultSwitches := []string{"switch-node-a", "switch-node-b"} + + tc := []struct { + name string + service *v1.Service + configs []lbConfig + nodeInfos []nodeInfo + expected []ovnlb.LB + }{ + { + name: "two tcp services, single stack", + service: defaultService, + configs: []lbConfig{ + { + vips: []string{"1.2.3.4"}, + protocol: v1.ProtocolTCP, + inport: 80, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1", "192.168.0.2"}, + Port: 8080, + }, + }, + { + vips: []string{"1.2.3.4"}, + protocol: v1.ProtocolTCP, + inport: 443, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + Port: 8043, + }, + }, + }, + nodeInfos: defaultNodes, + expected: []ovnlb.LB{ + { + Name: fmt.Sprintf("Service_%s/%s_TCP_cluster", namespace, name), + Protocol: "TCP", + ExternalIDs: defaultExternalIDs, + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"1.2.3.4", 80}, + Targets: []ovnlb.Addr{{"192.168.0.1", 8080}, {"192.168.0.2", 8080}}, + }, + { + Source: ovnlb.Addr{"1.2.3.4", 443}, + Targets: []ovnlb.Addr{{"192.168.0.1", 8043}}, + }, + }, + + Routers: defaultRouters, + Switches: defaultSwitches, + }, + }, + }, + { + name: "tcp / udp services, single stack", + service: defaultService, + configs: []lbConfig{ + { + vips: []string{"1.2.3.4"}, + protocol: v1.ProtocolTCP, + inport: 80, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1", "192.168.0.2"}, + Port: 8080, + }, + }, + { + vips: []string{"1.2.3.4"}, + protocol: v1.ProtocolUDP, + inport: 443, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + Port: 8043, + }, + }, + }, + nodeInfos: defaultNodes, + expected: []ovnlb.LB{ + { + Name: fmt.Sprintf("Service_%s/%s_TCP_cluster", namespace, name), + Protocol: "TCP", + ExternalIDs: defaultExternalIDs, + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"1.2.3.4", 80}, + Targets: []ovnlb.Addr{{"192.168.0.1", 8080}, {"192.168.0.2", 8080}}, + }, + }, + + Routers: defaultRouters, + Switches: defaultSwitches, + }, + { + Name: fmt.Sprintf("Service_%s/%s_UDP_cluster", namespace, name), + Protocol: "UDP", + ExternalIDs: defaultExternalIDs, + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"1.2.3.4", 443}, + Targets: []ovnlb.Addr{{"192.168.0.1", 8043}}, + }, + }, + + Routers: defaultRouters, + Switches: defaultSwitches, + }, + }, + }, + { + name: "dual stack", + service: defaultService, + configs: []lbConfig{ + { + vips: []string{"1.2.3.4", "fe80::1"}, + protocol: v1.ProtocolTCP, + inport: 80, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1", "192.168.0.2"}, + V6IPs: []string{"fe90::1", "fe91::1"}, + Port: 8080, + }, + }, + { + vips: []string{"1.2.3.4", "fe80::1"}, + protocol: v1.ProtocolTCP, + inport: 443, + eps: util.LbEndpoints{ + V4IPs: []string{"192.168.0.1"}, + V6IPs: []string{"fe90::1"}, + Port: 8043, + }, + }, + }, + nodeInfos: defaultNodes, + expected: []ovnlb.LB{ + { + Name: fmt.Sprintf("Service_%s/%s_TCP_cluster", namespace, name), + Protocol: "TCP", + ExternalIDs: defaultExternalIDs, + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"1.2.3.4", 80}, + Targets: []ovnlb.Addr{{"192.168.0.1", 8080}, {"192.168.0.2", 8080}}, + }, + { + Source: ovnlb.Addr{"fe80::1", 80}, + Targets: []ovnlb.Addr{{"fe90::1", 8080}, {"fe91::1", 8080}}, + }, + { + Source: ovnlb.Addr{"1.2.3.4", 443}, + Targets: []ovnlb.Addr{{"192.168.0.1", 8043}}, + }, + { + Source: ovnlb.Addr{"fe80::1", 443}, + Targets: []ovnlb.Addr{{"fe90::1", 8043}}, + }, + }, + + Routers: defaultRouters, + Switches: defaultSwitches, + }, + }, + }, + } + + for i, tt := range tc { + t.Run(fmt.Sprintf("%d_%s", i, tt.name), func(t *testing.T) { + actual := buildClusterLBs(tt.service, tt.configs, tt.nodeInfos) + assert.Equal(t, tt.expected, actual) + }) + } +} + +func Test_buildPerNodeLBs(t *testing.T) { + oldClusterSubnet := globalconfig.Default.ClusterSubnets + oldGwMode := globalconfig.Gateway.Mode + defer func() { + globalconfig.Gateway.Mode = oldGwMode + globalconfig.Default.ClusterSubnets = oldClusterSubnet + }() + _, cidr4, _ := net.ParseCIDR("10.128.0.0/16") + _, cidr6, _ := net.ParseCIDR("fe00::/64") + globalconfig.Default.ClusterSubnets = []globalconfig.CIDRNetworkEntry{{cidr4, 26}, {cidr6, 26}} + + name := "foo" + namespace := "testns" + + defaultService := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + }, + } + + defaultNodes := []nodeInfo{ + { + name: "node-a", + nodeIPs: []string{"10.0.0.1"}, + gatewayRouterName: "gr-node-a", + switchName: "switch-node-a", + podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.0.0"), Mask: net.CIDRMask(24, 32)}}, + }, + { + name: "node-b", + nodeIPs: []string{"10.0.0.2"}, + gatewayRouterName: "gr-node-b", + switchName: "switch-node-b", + podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.1.0"), Mask: net.CIDRMask(24, 32)}}, + }, + } + + defaultExternalIDs := map[string]string{ + "k8s.ovn.org/kind": "Service", + "k8s.ovn.org/owner": fmt.Sprintf("%s/%s", namespace, name), + } + + //defaultRouters := []string{"gr-node-a", "gr-node-b"} + //defaultSwitches := []string{"switch-node-a", "switch-node-b"} + + tc := []struct { + name string + service *v1.Service + configs []lbConfig + expectedShared []ovnlb.LB + expectedLocal []ovnlb.LB + }{ + { + name: "host-network pod", + service: defaultService, + configs: []lbConfig{ + { + vips: []string{"1.2.3.4"}, + protocol: v1.ProtocolTCP, + inport: 80, + eps: util.LbEndpoints{ + V4IPs: []string{"10.0.0.1"}, + Port: 8080, + }, + }, + }, + expectedShared: []ovnlb.LB{ + { + Name: "Service_testns/foo_TCP_node_router_node-a", + ExternalIDs: defaultExternalIDs, + Routers: []string{"gr-node-a"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"1.2.3.4", 80}, + Targets: []ovnlb.Addr{{"169.254.169.2", 8080}}, + }, + }, + }, + { + Name: "Service_testns/foo_TCP_node_switch_node-a_merged", + ExternalIDs: defaultExternalIDs, + Routers: []string{"gr-node-b"}, + Switches: []string{"switch-node-a", "switch-node-b"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"1.2.3.4", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + }, + }, + }, + }, + { + name: "nodeport service, standard pod", + service: defaultService, + configs: []lbConfig{ + { + vips: []string{"node"}, + protocol: v1.ProtocolTCP, + inport: 80, + eps: util.LbEndpoints{ + V4IPs: []string{"10.128.0.2"}, + Port: 8080, + }, + }, + }, + expectedShared: []ovnlb.LB{ + { + Name: "Service_testns/foo_TCP_node_router+switch_node-a", + ExternalIDs: defaultExternalIDs, + Routers: []string{"gr-node-a"}, + Switches: []string{"switch-node-a"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"10.0.0.1", 80}, + Targets: []ovnlb.Addr{{"10.128.0.2", 8080}}, + }, + }, + }, + { + Name: "Service_testns/foo_TCP_node_router+switch_node-b", + ExternalIDs: defaultExternalIDs, + Routers: []string{"gr-node-b"}, + Switches: []string{"switch-node-b"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"10.0.0.2", 80}, + Targets: []ovnlb.Addr{{"10.128.0.2", 8080}}, + }, + }, + }, + }, + expectedLocal: []ovnlb.LB{ + { + Name: "Service_testns/foo_TCP_node_switch_node-a", + ExternalIDs: defaultExternalIDs, + Switches: []string{"switch-node-a"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"10.0.0.1", 80}, + Targets: []ovnlb.Addr{{"10.128.0.2", 8080}}, + }, + }, + }, + { + Name: "Service_testns/foo_TCP_node_switch_node-b", + ExternalIDs: defaultExternalIDs, + Switches: []string{"switch-node-b"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"10.0.0.2", 80}, + Targets: []ovnlb.Addr{{"10.128.0.2", 8080}}, + }, + }, + }, + }, + }, + { + name: "nodeport service, host-network pod", + service: defaultService, + configs: []lbConfig{ + { + vips: []string{"192.168.0.1"}, + protocol: v1.ProtocolTCP, + inport: 80, + eps: util.LbEndpoints{ + V4IPs: []string{"10.0.0.1"}, + Port: 8080, + }, + }, + { + vips: []string{"node"}, + protocol: v1.ProtocolTCP, + inport: 80, + eps: util.LbEndpoints{ + V4IPs: []string{"10.0.0.1"}, + Port: 8080, + }, + }, + }, + expectedShared: []ovnlb.LB{ + { + Name: "Service_testns/foo_TCP_node_router_node-a", + ExternalIDs: defaultExternalIDs, + Routers: []string{"gr-node-a"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"192.168.0.1", 80}, + Targets: []ovnlb.Addr{{"169.254.169.2", 8080}}, + }, + { + Source: ovnlb.Addr{"10.0.0.1", 80}, + Targets: []ovnlb.Addr{{"169.254.169.2", 8080}}, + }, + }, + }, + { + Name: "Service_testns/foo_TCP_node_switch_node-a", + ExternalIDs: defaultExternalIDs, + Switches: []string{"switch-node-a"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"192.168.0.1", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + { + Source: ovnlb.Addr{"10.0.0.1", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + }, + }, + { + Name: "Service_testns/foo_TCP_node_router+switch_node-b", + ExternalIDs: defaultExternalIDs, + Routers: []string{"gr-node-b"}, + Switches: []string{"switch-node-b"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"192.168.0.1", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + { + Source: ovnlb.Addr{"10.0.0.2", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + }, + }, + }, + expectedLocal: []ovnlb.LB{ + { + Name: "Service_testns/foo_TCP_node_switch_node-a", + ExternalIDs: defaultExternalIDs, + Switches: []string{"switch-node-a"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"192.168.0.1", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + { + Source: ovnlb.Addr{"10.0.0.1", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + }, + }, + { + Name: "Service_testns/foo_TCP_node_switch_node-b", + ExternalIDs: defaultExternalIDs, + Switches: []string{"switch-node-b"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"192.168.0.1", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + { + Source: ovnlb.Addr{"10.0.0.2", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + }, + }, + }, + }, + { + // The most complicated case + name: "nodeport service, host-network pod, ExternalTrafficPolicy", + service: defaultService, + configs: []lbConfig{ + { + vips: []string{"192.168.0.1"}, + protocol: v1.ProtocolTCP, + inport: 80, + eps: util.LbEndpoints{ + V4IPs: []string{"10.0.0.1"}, + Port: 8080, + }, + }, + { + vips: []string{"node"}, + protocol: v1.ProtocolTCP, + inport: 80, + externalTrafficLocal: true, + eps: util.LbEndpoints{ + V4IPs: []string{"10.0.0.1"}, + Port: 8080, + }, + }, + }, + expectedShared: []ovnlb.LB{ + // node-a has endpoints: 3 load balancers + // router clusterip + // router nodeport + // switch clusterip + nodeport + { + Name: "Service_testns/foo_TCP_node_router_node-a", + ExternalIDs: defaultExternalIDs, + Routers: []string{"gr-node-a"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"192.168.0.1", 80}, + Targets: []ovnlb.Addr{{"169.254.169.2", 8080}}, + }, + }, + }, + { + Name: "Service_testns/foo_TCP_node_local_router_node-a", + ExternalIDs: defaultExternalIDs, + Routers: []string{"gr-node-a"}, + Opts: ovnlb.LBOpts{SkipSNAT: true}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"10.0.0.1", 80}, + Targets: []ovnlb.Addr{{"169.254.169.2", 8080}}, + }, + }, + }, + { + Name: "Service_testns/foo_TCP_node_switch_node-a", + ExternalIDs: defaultExternalIDs, + Switches: []string{"switch-node-a"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"192.168.0.1", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + { + Source: ovnlb.Addr{"10.0.0.1", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + }, + }, + + // node-b has no service, 3 lbs + // router clusterip + // router nodeport = empty + // switch clusterip + nodeport + { + Name: "Service_testns/foo_TCP_node_router_node-b", + ExternalIDs: defaultExternalIDs, + Routers: []string{"gr-node-b"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"192.168.0.1", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + { + Source: ovnlb.Addr{"10.0.0.2", 80}, + Targets: []ovnlb.Addr{}, + }, + }, + }, + { + Name: "Service_testns/foo_TCP_node_switch_node-b", + ExternalIDs: defaultExternalIDs, + Switches: []string{"switch-node-b"}, + Protocol: "TCP", + Rules: []ovnlb.LBRule{ + { + Source: ovnlb.Addr{"192.168.0.1", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + { + Source: ovnlb.Addr{"10.0.0.2", 80}, + Targets: []ovnlb.Addr{{"10.0.0.1", 8080}}, + }, + }, + }, + }, + }, + } + + for i, tt := range tc { + t.Run(fmt.Sprintf("%d_%s", i, tt.name), func(t *testing.T) { + + if tt.expectedShared != nil { + globalconfig.Gateway.Mode = globalconfig.GatewayModeShared + actual := buildPerNodeLBs(tt.service, tt.configs, defaultNodes) + assert.Equal(t, tt.expectedShared, actual, "shared gateway mode not as expected") + } + + if tt.expectedLocal != nil { + globalconfig.Gateway.Mode = globalconfig.GatewayModeLocal + actual := buildPerNodeLBs(tt.service, tt.configs, defaultNodes) + assert.Equal(t, tt.expectedLocal, actual, "local gateway mode not as expected") + } + + }) + } +} diff --git a/go-controller/pkg/ovn/controller/services/node_tracker.go b/go-controller/pkg/ovn/controller/services/node_tracker.go new file mode 100644 index 0000000000..09d5dca2f0 --- /dev/null +++ b/go-controller/pkg/ovn/controller/services/node_tracker.go @@ -0,0 +1,213 @@ +package services + +import ( + "net" + "reflect" + "sort" + "sync" + + globalconfig "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + + v1 "k8s.io/api/core/v1" + coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +// nodeTracker watches all Node objects and maintains a cache of information relevant +// to service creation. If a new node is created, it requests a resync of all services, +// since need to apply those service's load balancers to the new node as well. +type nodeTracker struct { + sync.Mutex + + // nodes is the list of nodes we know about + // map of name -> info + nodes map[string]nodeInfo + + // resyncFn is the function to call so that all service are resynced + resyncFn func() +} + +type nodeInfo struct { + // the node's Name + name string + // The list of physical IPs the node has, as reported by the gatewayconf annotation + nodeIPs []string + // The pod network subnet(s) + podSubnets []net.IPNet + // the name of the node's GatewayRouter, or "" of non-existent + gatewayRouterName string + // The name of the node's switch - never empty + switchName string +} + +// returns a list of all ip blocks "assigned" to this node +// includes node IPs, still as a mask-1 net +func (ni *nodeInfo) nodeSubnets() []net.IPNet { + out := append([]net.IPNet{}, ni.podSubnets...) + for _, ipStr := range ni.nodeIPs { + ip := net.ParseIP(ipStr) + if ip := ip.To4(); ip != nil { + out = append(out, net.IPNet{ + IP: ip, + Mask: net.CIDRMask(32, 32), + }) + } else { + out = append(out, net.IPNet{ + IP: ip, + Mask: net.CIDRMask(128, 128), + }) + } + } + + return out +} + +func newNodeTracker(nodeInformer coreinformers.NodeInformer) *nodeTracker { + nt := &nodeTracker{ + nodes: map[string]nodeInfo{}, + } + + nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + node, ok := obj.(*v1.Node) + if !ok { + return + } + nt.updateNode(node) + }, + UpdateFunc: func(old, new interface{}) { + oldObj, ok := old.(*v1.Node) + if !ok { + return + } + newObj, ok := new.(*v1.Node) + if !ok { + return + } + // Make sure object was actually changed and not pending deletion + if oldObj.GetResourceVersion() == newObj.GetResourceVersion() || !newObj.GetDeletionTimestamp().IsZero() { + return + } + + nt.updateNode(newObj) + }, + DeleteFunc: func(obj interface{}) { + node, ok := obj.(*v1.Node) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.Errorf("couldn't understand non-tombstone object") + return + } + node, ok = tombstone.Obj.(*v1.Node) + if !ok { + klog.Errorf("couldn't understand tombstone object") + return + } + } + nt.removeNode(node.Name) + }, + }) + + return nt + +} + +// updateNodeInfo updates the node info cache, and syncs all services +// if it changed. +func (nt *nodeTracker) updateNodeInfo(nodeName, switchName, routerName string, nodeIPs []string, podSubnets []*net.IPNet) { + ni := nodeInfo{ + name: nodeName, + nodeIPs: nodeIPs, + podSubnets: make([]net.IPNet, 0, len(podSubnets)), + gatewayRouterName: routerName, + switchName: switchName, + } + for i := range podSubnets { + ni.podSubnets = append(ni.podSubnets, *podSubnets[i]) // de-pointer + } + + nt.Lock() + if existing, ok := nt.nodes[nodeName]; ok { + if reflect.DeepEqual(existing, ni) { + nt.Unlock() + return + } + } + + nt.nodes[nodeName] = ni + nt.Unlock() + + klog.Infof("Node %s switch + router changed, syncing services", nodeName) + // Resync all services + nt.resyncFn() +} + +// RemoveNode removes a node from the LB -> node mapper +// We don't need to re-sync here, because any stale LBs +// will eventually be cleaned up, and they don't have any cost. +func (nt *nodeTracker) removeNode(nodeName string) { + nt.Lock() + defer nt.Unlock() + + delete(nt.nodes, nodeName) +} + +// UpdateNode is called when a node's gateway router / switch / IPs have changed +// The switch exists when the HostSubnet annotation is set. +// The gateway router will exist sometime after the L3Gateway annotation is set. +func (nt *nodeTracker) updateNode(node *v1.Node) { + klog.V(2).Infof("Processing possible switch / router updates for node %s", node.Name) + hsn, err := util.ParseNodeHostSubnetAnnotation(node) + if err != nil || hsn == nil { + // usually normal; means the node's gateway hasn't been initialized yet + klog.Infof("Node %s has invalid / no HostSubnet annotations (probably waiting on initialization): %v", node.Name, err) + nt.removeNode(node.Name) + return + } + + switchName := node.Name + grName := "" + ips := []string{} + + // if the node has a gateway config, it will soon have a gateway router + // so, set the router name + gwConf, err := util.ParseNodeL3GatewayAnnotation(node) + if err != nil || gwConf == nil { + klog.Infof("Node %s has invalid / no gateway config: %v", node.Name, err) + } else if gwConf.Mode != globalconfig.GatewayModeDisabled { + grName = util.GetGatewayRouterFromNode(node.Name) + if gwConf.NodePortEnable || gwConf.Mode == globalconfig.GatewayModeShared { + for _, ip := range gwConf.IPAddresses { + ips = append(ips, ip.IP.String()) + } + } + } + + nt.updateNodeInfo( + node.Name, + switchName, + grName, + ips, + hsn, + ) +} + +// allNodes returns a list of all nodes (and their relevant information) +func (nt *nodeTracker) allNodes() []nodeInfo { + nt.Lock() + defer nt.Unlock() + + out := make([]nodeInfo, 0, len(nt.nodes)) + for _, node := range nt.nodes { + out = append(out, node) + } + + // Sort the returned list of nodes + // so that other operations that consume this data can just do a DeepEquals of things + // (e.g. LB routers + switches) without having to do set arithmetic + sort.SliceStable(out, func(i, j int) bool { return out[i].name < out[j].name }) + return out +} diff --git a/go-controller/pkg/ovn/controller/services/repair.go b/go-controller/pkg/ovn/controller/services/repair.go index 571247f41b..28be506e7d 100644 --- a/go-controller/pkg/ovn/controller/services/repair.go +++ b/go-controller/pkg/ovn/controller/services/repair.go @@ -2,154 +2,174 @@ package services import ( "fmt" + "sync" + "sync/atomic" "time" - "github.com/pkg/errors" - + globalconfig "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/acl" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/gateway" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" + ovnlb "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" + utilwait "k8s.io/apimachinery/pkg/util/wait" corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/retry" "k8s.io/klog/v2" ) -// Repair is a controller loop that can work as in one-shot or periodic mode -// it checks the OVN Load Balancers and delete the ones that doesn't exist in -// the Kubernetes Service Informer cache. -// Based on: -// https://raw.githubusercontent.com/kubernetes/kubernetes/release-1.19/pkg/registry/core/service/ipallocator/controller/repair.go -type Repair struct { - interval time.Duration - // serviceTracker tracks services and maps them to OVN LoadBalancers - serviceLister corelisters.ServiceLister - clusterPortGroupUUID string +// repair handles pre-sync and post-sync service cleanup. +// It has two phases: +// +// Pre-Sync: Delete any ovn Load_Balancer rows that are "owned" by a Kubernetes Service +// that doesn't exist anymore +// +// Post-sync: After every service has been synced at least once, delete any legacy load-balancers +// +// We need to execute in two phases so that we don't disrupt any vips being handled by +// legacy load balancers before we've had a change to migrate those vips. +type repair struct { + sync.Mutex + serviceLister corelisters.ServiceLister + + // We want to run some functions after every service is successfully synced, so populate this + // list with every service that should be in the informer queue before we start the ServiceController + // workers. + unsyncedServices sets.String + + // Really a boolean, but an int32 for atomicity purposes + semLegacyLBsDeleted uint32 } // NewRepair creates a controller that periodically ensures that there is no stale data in OVN -func NewRepair(interval time.Duration, serviceLister corelisters.ServiceLister, clusterPortGroupUUID string) *Repair { - return &Repair{ - interval: interval, - serviceLister: serviceLister, - clusterPortGroupUUID: clusterPortGroupUUID, +func newRepair(serviceLister corelisters.ServiceLister) *repair { + return &repair{ + serviceLister: serviceLister, + unsyncedServices: sets.String{}, } } -// runOnce verifies the state of the cluster OVN LB VIP allocations and returns an error if an unrecoverable problem occurs. -func (r *Repair) runOnce() error { +// runBeforeSync performs some cleanup of stale LBs and other miscellaneous setup. +func (r *repair) runBeforeSync(clusterPortGroupUUID string) { + // no need to lock, single-threaded. + startTime := time.Now() klog.V(4).Infof("Starting repairing loop for services") defer func() { klog.V(4).Infof("Finished repairing loop for services: %v", time.Since(startTime)) }() - // Obtain all the load balancers UUID - ovnLBCache := make(map[v1.Protocol][]string) - // We have different loadbalancers per protocol - protocols := []v1.Protocol{v1.ProtocolSCTP, v1.ProtocolTCP, v1.ProtocolUDP} - // ClusterIP OVN load balancers - for _, p := range protocols { - lbUUID, err := loadbalancer.GetOVNKubeLoadBalancer(p) + // Ensure unidling is enabled + if globalconfig.Kubernetes.OVNEmptyLbEvents { + _, _, err := util.RunOVNNbctl("set", "nb_global", ".", "options:controller_event=true") if err != nil { - return errors.Wrapf(err, "Failed to get Cluster IP OVN load balancer for protocol %s", p) - } - ovnLBCache[p] = append(ovnLBCache[p], lbUUID) - } - // NodePort, ExternalIPs, Ingress OVN load balancers as well as worker load balancers - gatewayRouters, _, err := gateway.GetOvnGateways() - if err != nil { - klog.V(4).Infof("Failed to get gateway routers due to (%v). Skipping repairing OVN GR Load balancers", err) - } else { - for _, p := range protocols { - for _, gatewayRouter := range gatewayRouters { - lbUUID, err := gateway.GetGatewayLoadBalancer(gatewayRouter, p) - if err != nil { - if err != gateway.OVNGatewayLBIsEmpty { - klog.V(5).Infof("Failed to get OVN GR: %s load balancer for protocol %s, err: %v", - gatewayRouter, p, err) - } - } else { - ovnLBCache[p] = append(ovnLBCache[p], lbUUID) - } - workerNode := util.GetWorkerFromGatewayRouter(gatewayRouter) - workerLB, err := loadbalancer.GetWorkerLoadBalancer(workerNode, p) - if err != nil { - if err != gateway.OVNGatewayLBIsEmpty { - klog.V(5).Infof("Failed to get OVN Worker: %s load balancer for protocol %s, err: %v", - workerNode, p, err) - } - continue - } - ovnLBCache[p] = append(ovnLBCache[p], workerLB) - } + klog.Error("Unable to enable controller events. Unidling not possible") } } - // Idling load balancers - for _, p := range protocols { - lb, err := loadbalancer.GetOVNKubeIdlingLoadBalancer(p) - if err != nil { - return errors.Wrapf(err, "Failed to get Idling OVN load balancer for protocol %s", p) - } - ovnLBCache[p] = append(ovnLBCache[p], lb) + // Build a list of every service existing + // After every service has been synced, then we'll execute runAfterSync + services, _ := r.serviceLister.List(labels.Everything()) + for _, service := range services { + key, _ := cache.MetaNamespaceKeyFunc(service) + r.unsyncedServices.Insert(key) } - // Get Kubernetes Service state - svcVIPsProtocolMap := sets.NewString() - services, err := r.serviceLister.List(labels.Everything()) + // Find all load-balancers associated with Services + lbCache, err := ovnlb.GetLBCache() if err != nil { - return errors.Wrapf(err, "Failed to list Services from the cache") + klog.Errorf("Failed to get load_balancer cache: %v", err) } - for _, svc := range services { - for _, ip := range util.GetClusterIPs(svc) { - for _, svcPort := range svc.Spec.Ports { - vip := util.JoinHostPortInt32(ip, svcPort.Port) - key := virtualIPKey(vip, svcPort.Protocol) - svcVIPsProtocolMap.Insert(key) - } + existingLBs := lbCache.Find(map[string]string{"k8s.ovn.org/kind": "Service"}) + + // Look for any load balancers whose Service no longer exists in the apiserver + staleLBs := []string{} + for _, lb := range existingLBs { + // Extract namespace + name, look to see if it exists + owner := lb.ExternalIDs["k8s.ovn.org/owner"] + namespace, name, err := cache.SplitMetaNamespaceKey(owner) + if err != nil || namespace == "" { + klog.Warningf("Service LB %#v has unreadable owner, deleting", lb) + staleLBs = append(staleLBs, lb.UUID) } - } - // Reconcile with OVN state - // Obtain all the VIPs present in the OVN LoadBalancers - // and delete the ones that are not present in Kubernetes - for _, p := range protocols { - for _, lb := range ovnLBCache[p] { - vips, err := loadbalancer.GetLoadBalancerVIPs(lb) - if err != nil { - klog.V(4).Infof("Failed to get vips for %s load balancer %s, err: %v", p, lb, err) - continue - } - txn := util.NewNBTxn() - for vip := range vips { - key := virtualIPKey(vip, p) - // Virtual IP and protocol doesn't belong to a Kubernetes service - if !svcVIPsProtocolMap.Has(key) { - klog.Infof("Deleting non-existing Kubernetes vip %s from OVN %s load balancer %s", vip, p, lb) - if err := loadbalancer.DeleteLoadBalancerVIP(txn, lb, vip); err != nil { - klog.V(4).Infof("Failed to delete %s load balancer vips %s for %s, err: %v", p, vip, lb, err) - } - } - } - stdout, stderr, err := txn.Commit() - if err != nil { - return fmt.Errorf("error in deleting %s load balancer %s stale vips, "+ - "stdout: %q, stderr: %q, err: %v", - p, lb, stdout, stderr, err) - } + _, err = r.serviceLister.Services(namespace).Get(name) + if apierrors.IsNotFound(err) { + klog.V(5).Infof("Found stale service LB %#v", lb) + staleLBs = append(staleLBs, lb.UUID) } } + // Delete those stale load balancers + if err := ovnlb.DeleteLBs(nil, staleLBs); err != nil { + klog.Errorf("Failed to delete stale LBs: %v", err) + } + klog.V(2).Infof("Deleted %d stale service LBs", len(staleLBs)) + // Remove existing reject rules. They are not used anymore // given the introduction of idling loadbalancers - err = acl.PurgeRejectRules(r.clusterPortGroupUUID) + err = acl.PurgeRejectRules(clusterPortGroupUUID) if err != nil { klog.Errorf("Failed to purge existing reject rules: %v", err) + } +} + +// serviceSynced is called by a ServiceController worker when it has successfully +// applied a service. +// If all services have successfully synced at least once, kick off +// runAfterSync() +func (r *repair) serviceSynced(key string) { + r.Lock() + defer r.Unlock() + if len(r.unsyncedServices) == 0 { + return + } + delete(r.unsyncedServices, key) + if len(r.unsyncedServices) == 0 { + go r.runAfterSync() // run in a goroutine so we don't block the ServiceController + } +} + +// runAfterSync is called sometime after every existing service is successfully synced at least once +// It deletes all legacy load balancers. +func (r *repair) runAfterSync() { + _ = utilwait.ExponentialBackoff(retry.DefaultBackoff, func() (bool, error) { + klog.Infof("Running Service post-sync cleanup") + err := r.deleteLegacyLBs() + if err != nil { + klog.Warningf("Failed to delete legacy LBs: %v", err) + return false, nil + } + return true, nil + }) +} + +func (r *repair) deleteLegacyLBs() error { + // Find all load-balancers associated with Services + legacyLBs, err := findLegacyLBs() + if err != nil { + klog.Errorf("Failed to list existing load balancers: %v", err) return err } + + klog.V(2).Infof("Deleting %d legacy LBs", len(legacyLBs)) + toDelete := make([]string, 0, len(legacyLBs)) + for _, lb := range legacyLBs { + toDelete = append(toDelete, lb.UUID) + } + if err := ovnlb.DeleteLBs(nil, toDelete); err != nil { + return fmt.Errorf("failed to delete LBs: %w", err) + } + atomic.StoreUint32(&r.semLegacyLBsDeleted, 1) return nil } + +// legacyLBsDeleted returns true if we've run the post-sync repair +// and there are no more legacy LBs, so we can stop searching +// for them in the services handler. +func (r *repair) legacyLBsDeleted() bool { + return atomic.LoadUint32(&r.semLegacyLBsDeleted) > 0 +} diff --git a/go-controller/pkg/ovn/controller/services/repair_test.go b/go-controller/pkg/ovn/controller/services/repair_test.go deleted file mode 100644 index 802669899b..0000000000 --- a/go-controller/pkg/ovn/controller/services/repair_test.go +++ /dev/null @@ -1,419 +0,0 @@ -package services - -import ( - "testing" - - ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "k8s.io/client-go/informers" - coreinformers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes/fake" -) - -const ( - tcpLBUUID string = "1a3dfc82-2749-4931-9190-c30e7c0ecea3" - udpLBUUID string = "6d3142fc-53e8-4ac1-88e6-46094a5a9957" - sctpLBUUID string = "0514c521-a120-4756-aec6-883fe5db7139" - grTcpLBUUID string = "001c2ec6-2f32-11eb-9bc2-a8a1590cda29" - grUdpLBUUID string = "05c55ae6-2f32-11eb-822e-a8a1590cda29" - grSctpLBUUID string = "0ac92874-2f32-11eb-8ca0-a8a1590cda29" - workerTCPLBUUID string = "2095292c-adb4-11eb-8529-0242ac130003" - workerUDPLBUUID string = "2b662964-adb4-11eb-8529-0242ac130003" - workerSCTPLBUUID string = "50738e40-adb4-11eb-8529-0242ac130003" - idlingTCPLB string = "a64d5efe-adb4-11eb-8529-0242ac130003" - idlingUDPLB string = "bd58fa04-adb4-11eb-8529-0242ac130003" - idlingSCTPLB string = "c90b1940-adb4-11eb-8529-0242ac130003" -) - -func newServiceInformer() coreinformers.ServiceInformer { - client := fake.NewSimpleClientset() - informerFactory := informers.NewSharedInformerFactory(client, 0) - return informerFactory.Core().V1().Services() -} - -func TestRepair_Empty(t *testing.T) { - serviceInformer := newServiceInformer() - r := &Repair{ - interval: 0, - serviceLister: serviceInformer.Lister(), - } - // Expected OVN commands - fexec := ovntest.NewFakeExec() - initializeClusterIPLBs(fexec) - // OVN is empty - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + sctpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grSctpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerSCTPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingSCTPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + tcpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grTcpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerTCPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingTCPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + udpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grUdpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerUDPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingUDPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --columns=_uuid --format=csv --data=bare --no-headings find acl action=reject", - Output: "", - }) - - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - - if err := r.runOnce(); err != nil { - t.Errorf("Unexpected error: %v", err) - } -} - -func TestRepair_OVNStaleData(t *testing.T) { - serviceInformer := newServiceInformer() - r := &Repair{ - interval: 0, - serviceLister: serviceInformer.Lister(), - } - fexec := ovntest.NewLooseCompareFakeExec() - initializeClusterIPLBs(fexec) - // There are remaining OVN LB that doesn't exist in Kubernetes - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + sctpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grSctpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerSCTPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingSCTPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + tcpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grTcpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerTCPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingTCPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + udpLBUUID + " vips", - Output: `{"10.96.0.10:53"="10.244.2.3:53,10.244.2.5:53", "10.96.0.10:9153"="10.244.2.3:9153,10.244.2.5:9153", "10.96.0.1:443"="172.19.0.3:6443"}`, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grUdpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerUDPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingUDPLB + " vips", - Output: "", - }) - // The repair loop must delete the remaining entries in OVN - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer " + udpLBUUID + " vips \"10.96.0.10:53\" -- --if-exists remove load_balancer " + udpLBUUID + " vips \"10.96.0.10:9153\" -- --if-exists remove load_balancer " + udpLBUUID + " vips \"10.96.0.1:443\"", - LooseBatchCompare: true, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --columns=_uuid --format=csv --data=bare --no-headings find acl action=reject", - Output: "", - }) - - // The repair loop must delete them - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - - if err := r.runOnce(); err != nil { - t.Errorf("Unexpected error: %v", err) - } -} - -func TestRepair_OVNSynced(t *testing.T) { - // Initialize informer cache - serviceInformer := newServiceInformer() - serviceStore := serviceInformer.Informer().GetStore() - serviceStore.Add(createService("svc1", "10.96.0.10", 80)) - serviceStore.Add(createService("svc2", "fd00:10:96::1", 80)) - - r := &Repair{ - interval: 0, - serviceLister: serviceInformer.Lister(), - } - // Expected OVN commands - fexec := ovntest.NewLooseCompareFakeExec() - initializeClusterIPLBs(fexec) - - // OVN database is in Sync no operation expected - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + sctpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grSctpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerSCTPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingSCTPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + tcpLBUUID + " vips", - Output: `{"10.96.0.10:80"="10.0.0.2:3456,10.0.0.3:3456", "[fd00:10:96::1]:80"="[2001:db8::1]:3456,[2001:db8::2]:3456"}`, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grTcpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerTCPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingTCPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + udpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grUdpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerUDPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingUDPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --columns=_uuid --format=csv --data=bare --no-headings find acl action=reject", - Output: "", - }) - - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - - if err := r.runOnce(); err != nil { - t.Errorf("Unexpected error: %v", err) - } -} - -func TestRepair_OVNMissingService(t *testing.T) { - // Initialize informer cache - serviceInformer := newServiceInformer() - serviceStore := serviceInformer.Informer().GetStore() - serviceStore.Add(createService("svc1", "10.96.0.10", 80)) - serviceStore.Add(createService("svc2", "fd00:10:96::1", 80)) - - r := &Repair{ - interval: 0, - serviceLister: serviceInformer.Lister(), - } - fexec := ovntest.NewFakeExec() - initializeClusterIPLBs(fexec) - - // OVN database is in Sync no operation expected - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + sctpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grSctpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerSCTPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingSCTPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + tcpLBUUID + " vips", - Output: `{"10.96.0.10:80"="10.0.0.2:3456,10.0.0.3:3456"}`, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grTcpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerTCPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingTCPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + udpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + grUdpLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + workerUDPLBUUID + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer " + idlingUDPLB + " vips", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --columns=_uuid --format=csv --data=bare --no-headings find acl action=reject", - Output: "", - }) - - // The repair loop must do nothing, the controller will add the new service - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - - if err := r.runOnce(); err != nil { - t.Errorf("Unexpected error: %v", err) - } -} - -func initializeClusterIPLBs(fexec *ovntest.FakeExec) { - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-sctp=yes", - Output: sctpLBUUID, - }) - - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: tcpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-udp=yes", - Output: udpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: "gateway1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:SCTP_lb_gateway_router=gateway1", - Output: grSctpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-sctp=gateway1", - Output: workerSCTPLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=gateway1", - Output: grTcpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-tcp=gateway1", - Output: workerTCPLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:UDP_lb_gateway_router=gateway1", - Output: grUdpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-udp=gateway1", - Output: workerUDPLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-sctp=yes", - Output: idlingSCTPLB, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingTCPLB, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-udp=yes", - Output: idlingUDPLB, - }) -} - -func createService(name, ip string, port int) *v1.Service { - return &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "nsname"}, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeClusterIP, - ClusterIP: ip, - ClusterIPs: []string{ip}, - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{{ - Port: int32(port), - Protocol: v1.ProtocolTCP, - TargetPort: intstr.FromInt(3456), - }}, - }, - } -} diff --git a/go-controller/pkg/ovn/controller/services/service_tracker.go b/go-controller/pkg/ovn/controller/services/service_tracker.go deleted file mode 100644 index 2e061838e0..0000000000 --- a/go-controller/pkg/ovn/controller/services/service_tracker.go +++ /dev/null @@ -1,167 +0,0 @@ -package services - -import ( - "k8s.io/apimachinery/pkg/util/sets" - "strings" - "sync" - - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - v1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" -) - -// serviceTrackerKey returns a string used for the tracker index. -func serviceTrackerKey(name, namespace string) string { return name + "/" + namespace } - -// virtualIPKey returns a string used for the virtual IPs index. -// it accepts a join of ip and port and the protocol. -func virtualIPKey(vipAndPort string, protocol v1.Protocol) string { - return vipAndPort + "/" + string(protocol) -} - -// splitVirtualIPKey splits the VirtualIPKey from the service tracker in virtual ip and protocol -func splitVirtualIPKey(key string) (string, v1.Protocol) { - parts := strings.Split(key, "/") - return parts[0], v1.Protocol(parts[1]) -} - -// loadbalancersPerVip is used to track vips to OVN load balancers -type loadbalancersPerVip map[string]string - -// serviceTracker tracks the services VIPs using the service name and namespace as key -// one service can have multiple VIPs, they are stored in the format IP:Port/Protocol -// The services allows mapping between Kubernetes Services and OVN LoadBalancer -type serviceTracker struct { - sync.Mutex - // holds a map of service mapping to map of vips and loadbalancers - virtualIPByService map[string]loadbalancersPerVip - hadEndpoints map[string]bool -} - -// newServiceTracker creates and initializes a new serviceTracker. -func newServiceTracker() *serviceTracker { - return &serviceTracker{ - virtualIPByService: map[string]loadbalancersPerVip{}, - hadEndpoints: map[string]bool{}, - } -} - -// updateService adds or updates the virtualIPs and endpoints of the Service -func (st *serviceTracker) updateService(name, namespace, virtualIP string, proto v1.Protocol, loadBalancer string) { - st.Lock() - defer st.Unlock() - - serviceNN := serviceTrackerKey(name, namespace) - key := virtualIPKey(virtualIP, proto) - - // check if the service already exists and create a new entry if it does not - vips, ok := st.virtualIPByService[serviceNN] - if !ok || vips == nil { - klog.V(5).Infof("Created service %s VIP %s %s on Service Tracker", serviceNN, virtualIP, proto) - st.virtualIPByService[serviceNN] = loadbalancersPerVip{key: loadBalancer} - return - } - // Update the service VIP with the new endpoints - vips[key] = loadBalancer - klog.V(5).Infof("Updated service %s VIP %s %s on Service Tracker", serviceNN, virtualIP, proto) -} - -// deleteService removes the set of virtual IPs tracked for the Service. -func (st *serviceTracker) deleteService(name, namespace string) { - st.Lock() - defer st.Unlock() - - serviceNN := serviceTrackerKey(name, namespace) - delete(st.virtualIPByService, serviceNN) - delete(st.hadEndpoints, serviceNN) - klog.V(5).Infof("Deleted service %s from Service Tracker", serviceNN) -} - -// deleteServiceVIP removes the virtual IP tracked for the Service. -func (st *serviceTracker) deleteServiceVIP(name, namespace, virtualIP string, proto v1.Protocol) { - st.Lock() - defer st.Unlock() - - serviceNN := serviceTrackerKey(name, namespace) - key := virtualIPKey(virtualIP, proto) - vips, ok := st.virtualIPByService[serviceNN] - if ok { - delete(vips, key) - klog.V(5).Infof("Deleted service %s VIP %s %s from Service Tracker", serviceNN, virtualIP, proto) - } -} - -// deleteServiceVIPs removes all the virtual IPs tracked for the Service. -func (st *serviceTracker) deleteServiceVIPs(name, namespace string, virtualIPs sets.String) { - for vipKey := range virtualIPs { - // the VIP is stored with the format IP:Port/Protocol - vip, proto := splitVirtualIPKey(vipKey) - st.deleteServiceVIP(name, namespace, vip, proto) - } -} - -// hasService return true if the service is being tracked -func (st *serviceTracker) hasService(name, namespace string) bool { - st.Lock() - defer st.Unlock() - - serviceNN := serviceTrackerKey(name, namespace) - _, ok := st.virtualIPByService[serviceNN] - return ok -} - -// hasServiceVIP return true if the VIP is being tracked for that service -func (st *serviceTracker) hasServiceVIP(name, namespace, virtualIP string, proto v1.Protocol) bool { - st.Lock() - defer st.Unlock() - - serviceNN := serviceTrackerKey(name, namespace) - key := virtualIPKey(virtualIP, proto) - - // check if the service already exists - vips, ok := st.virtualIPByService[serviceNN] - if !ok { - return false - } - _, ok = vips[key] - return ok -} - -// getService return the service VIPs associated to the service -func (st *serviceTracker) getService(name, namespace string) sets.String { - st.Lock() - defer st.Unlock() - - serviceNN := serviceTrackerKey(name, namespace) - if vips, ok := st.virtualIPByService[serviceNN]; ok { - klog.V(5).Infof("Obtained service %s on Service Tracker: %v", serviceNN, vips) - return sets.StringKeySet(vips) - } - return sets.NewString() -} - -// updateKubernetesService adds or updates the tracker from a Kubernetes service -// added for testing purposes -func (st *serviceTracker) updateKubernetesService(service *v1.Service, loadbalancer string) { - for _, ip := range util.GetClusterIPs(service) { - for _, svcPort := range service.Spec.Ports { - vip := util.JoinHostPortInt32(ip, svcPort.Port) - st.updateService(service.Name, service.Namespace, vip, svcPort.Protocol, loadbalancer) - } - } -} - -// GetLoadBalancer return the OVN LoadBalancer associated with a service VIP -func (st *serviceTracker) getLoadBalancer(name, namespace, vipProtocol string) string { - st.Lock() - defer st.Unlock() - serviceNN := serviceTrackerKey(name, namespace) - if vips, ok := st.virtualIPByService[serviceNN]; ok { - if lb, ok := vips[vipProtocol]; ok { - klog.V(5).Infof("Obtained load balancer: %s for service %s with vipProtocol: %s", - lb, serviceNN, vipProtocol) - return lb - } - } - return "" -} diff --git a/go-controller/pkg/ovn/controller/services/service_tracker_test.go b/go-controller/pkg/ovn/controller/services/service_tracker_test.go deleted file mode 100644 index a5186d911d..0000000000 --- a/go-controller/pkg/ovn/controller/services/service_tracker_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package services - -import ( - "testing" - - v1 "k8s.io/api/core/v1" -) - -func Test_serviceTracker_updateService(t *testing.T) { - tests := []struct { - name string - namespace string - proto v1.Protocol - vip string - }{ - // Create a new service - { - name: "svc1", - namespace: "ns1", - proto: v1.ProtocolTCP, - vip: "10.0.0.0:80", - }, - // Update the service endpoint - { - name: "svc1", - namespace: "ns1", - proto: v1.ProtocolTCP, - vip: "10.0.0.0:80", - }, - // Add a new VIP to the service - { - name: "svc1", - namespace: "ns1", - proto: v1.ProtocolTCP, - vip: "10.0.0.0:890", - }, - // Create another service - { - name: "svc2", - namespace: "ns1", - proto: v1.ProtocolUDP, - vip: "10.0.0.0:80", - }, - } - - st := newServiceTracker() - for _, tt := range tests { - st.updateService(tt.name, tt.namespace, tt.vip, tt.proto, "") - if !st.hasService(tt.name, tt.namespace) { - t.Fatalf("Error: service %s %s not updated", tt.name, tt.namespace) - } - if !st.hasServiceVIP(tt.name, tt.namespace, tt.vip, tt.proto) { - t.Fatalf("Error: service %s %s vip %s-%v not updated", tt.name, tt.namespace, tt.vip, tt.proto) - } - } - - // Delete a non existent service - st.deleteService("svc-fake", "ns1") - if st.hasService("svc-fake", "ns1") { - t.Fatalf("Service ns1 svc-fake should not exist") - } - // Delete service svc2 - st.deleteService("svc2", "ns1") - if st.hasService("svc2", "ns1") { - t.Fatalf("Service ns1 svc2 should not exist") - } - // Delete a non existent VIP (is a noop) - st.deleteServiceVIP("svc3", "ns1", "10.0.0.0:890", v1.ProtocolTCP) - // Delete vip 10.0.0.0:890-TCP on service svc1 - st.deleteServiceVIP("svc1", "ns1", "10.0.0.0:890", v1.ProtocolTCP) - if st.hasServiceVIP("svc1", "ns1", "10.0.0.0:890", v1.ProtocolTCP) { - t.Fatalf("Service ns1 svc2 should not exist") - } -} diff --git a/go-controller/pkg/ovn/controller/services/services_controller.go b/go-controller/pkg/ovn/controller/services/services_controller.go index c407090f9b..b6e33e7431 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller.go +++ b/go-controller/pkg/ovn/controller/services/services_controller.go @@ -2,21 +2,22 @@ package services import ( "fmt" + "reflect" + "sync" "time" libovsdbclient "github.com/ovn-org/libovsdb/client" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" + ovnlb "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - "github.com/pkg/errors" + "golang.org/x/time/rate" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" coreinformers "k8s.io/client-go/informers/core/v1" @@ -31,7 +32,6 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" - utilnet "k8s.io/utils/net" ) const ( @@ -50,7 +50,7 @@ func NewController(client clientset.Interface, nbClient libovsdbclient.Client, serviceInformer coreinformers.ServiceInformer, endpointSliceInformer discoveryinformers.EndpointSliceInformer, - clusterPortGroupUUID string, + nodeInformer coreinformers.NodeInformer, ) *Controller { klog.V(4).Info("Creating event broadcaster") broadcaster := record.NewBroadcaster() @@ -58,14 +58,12 @@ func NewController(client clientset.Interface, broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: controllerName}) - st := newServiceTracker() - c := &Controller{ client: client, nbClient: nbClient, - serviceTracker: st, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerName), + queue: workqueue.NewNamedRateLimitingQueue(newRatelimiter(100), controllerName), workerLoopPeriod: time.Second, + alreadyApplied: map[string][]ovnlb.LB{}, } // services @@ -93,7 +91,13 @@ func NewController(client clientset.Interface, c.eventRecorder = recorder // repair controller - c.repair = NewRepair(0, serviceInformer.Lister(), clusterPortGroupUUID) + c.repair = newRepair(serviceInformer.Lister()) + + // load balancers need to be applied to nodes, so + // we need to watch Node objects for changes. + c.nodeTracker = newNodeTracker(nodeInformer) + c.nodeTracker.resyncFn = c.RequestFullSync // Need to re-sync all services when a node gains its switch or GWR + c.nodesSynced = nodeInformer.Informer().HasSynced return c } @@ -108,9 +112,6 @@ type Controller struct { eventBroadcaster record.EventBroadcaster eventRecorder record.EventRecorder - // serviceTrack tracks services and map them to OVN LoadBalancers - serviceTracker *serviceTracker - // serviceLister is able to list/get services and is populated by the shared informer passed to serviceLister corelisters.ServiceLister // servicesSynced returns true if the service shared informer has been synced at least once. @@ -124,6 +125,8 @@ type Controller struct { // injection for testing. endpointSlicesSynced cache.InformerSynced + nodesSynced cache.InformerSynced + // Services that need to be updated. A channel is inappropriate here, // because it allows services with lots of pods to be serviced much // more often than services with few pods; it also would cause a @@ -135,12 +138,20 @@ type Controller struct { workerLoopPeriod time.Duration // repair contains a controller that keeps in sync OVN and Kubernetes services - repair *Repair + repair *repair + + // nodeTracker + nodeTracker *nodeTracker + + // alreadyApplied is a map of service key -> already applied configuration, so we can short-circuit + // if a service's config hasn't changed + alreadyApplied map[string][]ovnlb.LB + alreadyAppliedLock sync.Mutex } // Run will not return until stopCh is closed. workers determines how many // endpoints will be handled in parallel. -func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair bool) error { +func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair bool, clusterPortGroupUUID string) error { defer utilruntime.HandleCrash() defer c.queue.ShutDown() @@ -149,7 +160,7 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair bool) er // Wait for the caches to be synced klog.Info("Waiting for informer caches to sync") - if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.servicesSynced, c.endpointSlicesSynced) { + if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.servicesSynced, c.endpointSlicesSynced, c.nodesSynced) { return fmt.Errorf("error syncing cache") } @@ -157,10 +168,7 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair bool) er // Run the repair controller only once // it keeps in sync Kubernetes and OVN // and handles removal of stale data on upgrades - klog.Info("Remove stale OVN services") - if err := c.repair.runOnce(); err != nil { - klog.Errorf("Error repairing services: %v", err) - } + c.repair.runBeforeSync(clusterPortGroupUUID) } // Start the workers after the repair loop to avoid races klog.Info("Starting workers") @@ -188,7 +196,7 @@ func (c *Controller) processNextWorkItem() bool { } defer c.queue.Done(eKey) - err := c.syncServices(eKey.(string)) + err := c.syncService(eKey.(string)) c.handleErr(err, eKey) return true @@ -217,13 +225,19 @@ func (c *Controller) handleErr(err error, key interface{}) { utilruntime.HandleError(err) } -func (c *Controller) syncServices(key string) error { +// syncService ensures a given Service is correctly reflected in OVN. It does this by +// 1. Generating a high-level desired configuration +// 2. Converting the high-level configuration in to a list of exact OVN Load_Balancer objects +// 3. Reconciling those desired objects against the database. +// +// All Load_Balancer objects are tagged with their owner, so it's easy to find stale objects. +func (c *Controller) syncService(key string) error { startTime := time.Now() namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { return err } - klog.Infof("Processing sync for service %s on namespace %s ", name, namespace) + klog.Infof("Processing sync for service %s/%s", namespace, name) metrics.MetricSyncServiceCount.Inc() defer func() { @@ -238,32 +252,30 @@ func (c *Controller) syncServices(key string) error { if err != nil && !apierrors.IsNotFound(err) { return err } - // Get current state of the Service from the Service tracker - // These are the VIPs (ClusterIP:Port) that we have seen so far - // If the Service has updated the VIPs (has changed the Ports) - // and some were removed we have to delete those - // We need to create a map to not mutate the service tracker VIPs - vipsTracked := sets.NewString().Union(c.serviceTracker.getService(name, namespace)) - // Delete the Service VIPs from OVN if: + + // Delete the Service's LB(s) from OVN if: // - the Service was deleted from the cache (doesn't exist in Kubernetes anymore) // - the Service mutated to a new service Type that we don't handle (ExternalName, Headless) - if err != nil || !util.ServiceTypeHasClusterIP(service) || !util.IsClusterIPSet(service) { - err = deleteVIPsFromAllOVNBalancers(vipsTracked, name, namespace) - if err != nil { - // If the service wasn't found, don't panic sending an - // an event after cleaning it up - if service != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToDeleteOVNLoadBalancer", - "Error trying to delete the OVN LoadBalancer for Service %s/%s: %v", name, namespace, err) - } - return err + if err != nil || service == nil || !util.ServiceTypeHasClusterIP(service) || !util.IsClusterIPSet(service) { + service = &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } + + if err := ovnlb.EnsureLBs(util.ExternalIDsForObject(service), nil); err != nil { + return fmt.Errorf("failed to delete load balancers for service %s/%s: %w", + namespace, name, err) } - // Delete the Service form the Service Tracker - c.serviceTracker.deleteService(name, namespace) + + c.repair.serviceSynced(key) return nil } - klog.Infof("Creating service %s on namespace %s on OVN", name, namespace) + + // // The Service exists in the cache: update it in OVN + // // Get the endpoint slices associated to the Service esLabelSelector := labels.Set(map[string]string{ discovery.LabelServiceName: name, @@ -276,223 +288,62 @@ func (c *Controller) syncServices(key string) error { return err } - // If idling enabled and there are no endpoints, we need to move the VIP from the main loadbalancer - // ,that has the reject option for backends without endpoints, to the idling loadbalancer, that - // generates a needPods event. - // if not, we delete the vips from the idling lb and move them to their right place. - - vipProtocols := collectServiceVIPs(service) - if svcNeedsIdling(service.Annotations) && !util.HasValidEndpoint(service, endpointSlices) { - toRemoveFromNonIdling := sets.NewString() - for vipProtocol := range vipProtocols { - if c.serviceTracker.getLoadBalancer(name, namespace, vipProtocol) != loadbalancer.IdlingLoadBalancer { - toRemoveFromNonIdling.Insert(vipProtocol) - } - vipsTracked.Delete(vipProtocol) - } - // addServiceToIdlingBalancer adds the vips to service tracker - err = c.addServiceToIdlingBalancer(vipProtocols, service) - if err != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToAddToIdlingBalancer", - "Error trying to add to Idling LoadBalancer for Service %s/%s: %v", name, namespace, err) - return err + // Build the abstract LB configs for this service + perNodeConfigs, clusterConfigs := buildServiceLBConfigs(service, endpointSlices) + klog.V(5).Infof("Built service %s LB cluster-wide configs %#v", key, clusterConfigs) + klog.V(5).Infof("Built service %s LB per-node configs %#v", key, perNodeConfigs) + + // Convert the LB configs in to load-balancer objects + nodeInfos := c.nodeTracker.allNodes() + clusterLBs := buildClusterLBs(service, clusterConfigs, nodeInfos) + perNodeLBs := buildPerNodeLBs(service, perNodeConfigs, nodeInfos) + klog.V(3).Infof("Service %s has %d cluster-wide and %d per-node configs, making %d and %d load balancers", + key, len(clusterConfigs), len(perNodeConfigs), len(clusterLBs), len(perNodeLBs)) + lbs := append(clusterLBs, perNodeLBs...) + + // Short-circuit if nothing has changed + c.alreadyAppliedLock.Lock() + existingLBs, ok := c.alreadyApplied[key] + c.alreadyAppliedLock.Unlock() + if ok && reflect.DeepEqual(lbs, existingLBs) { + klog.V(3).Infof("Skipping no-op change for service %s", key) + } else { + // Actually apply load-balancers to OVN. + // + // Note: this may fail if a node was deleted between listing nodes and applying. + // If so, this will fail and we will resync. + if err := ovnlb.EnsureLBs(util.ExternalIDsForObject(service), lbs); err != nil { + return fmt.Errorf("failed to ensure service %s load balancers: %w", key, err) } - err = deleteVIPsFromNonIdlingOVNBalancers(toRemoveFromNonIdling, name, namespace) - if err != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToDeleteOVNLoadBalancer", - "Error trying to delete the OVN LoadBalancer while setting up Idling for Service %s/%s: %v", name, namespace, err) - return err - } - // at this point we have processed all vips we've found in the service - // so the remaining ones that we had in the vipsTracked variable should be deleted - // We remove them from OVN and from the tracker - err = deleteVIPsFromAllOVNBalancers(vipsTracked, name, namespace) - if err != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToDeleteOVNLoadBalancer", - "Error trying to delete the OVN LoadBalancer for Service %s/%s: %v", name, namespace, err) - return err - } - // Delete the Service VIP from the Service Tracker - c.serviceTracker.deleteServiceVIPs(name, namespace, vipsTracked) - return nil - } - toRemoveFromIdling := sets.NewString() - for vipProtocol := range vipProtocols { - foundLb := c.serviceTracker.getLoadBalancer(name, namespace, vipProtocol) - // if vip was on an idling load balancer and we get here, we know we need to clean up - // if the value is empty (unknown what load balancer the vip may have existed on) also need to clean up - if len(foundLb) == 0 || foundLb == loadbalancer.IdlingLoadBalancer { - toRemoveFromIdling.Insert(vipProtocol) - } - } - err = deleteVIPsFromIdlingBalancer(toRemoveFromIdling, name, namespace) - if err != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToDeleteOVNLoadBalancer", - "Error trying to delete the idling OVN LoadBalancer for Service %s/%s: %v", name, namespace, err) - return err + c.alreadyAppliedLock.Lock() + c.alreadyApplied[key] = lbs + c.alreadyAppliedLock.Unlock() } - // Iterate over the ClusterIPs and Ports fields to create the corresponding OVN loadbalancers - for _, ip := range util.GetClusterIPs(service) { - family := v1.IPv4Protocol - if utilnet.IsIPv6String(ip) { - family = v1.IPv6Protocol - } - for _, svcPort := range service.Spec.Ports { - // ClusterIP - clusterLB, err := loadbalancer.GetOVNKubeLoadBalancer(svcPort.Protocol) - if err != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToGetOVNLoadBalancer", - "Error trying to obtain the OVN LoadBalancer for Service %s/%s: %v", name, namespace, err) - return err - } - // create the vip = ClusterIP:Port - vip := util.JoinHostPortInt32(ip, svcPort.Port) - klog.V(4).Infof("Updating service %s/%s with VIP %s %s", name, namespace, vip, svcPort.Protocol) - // get the endpoints associated to the vip - eps := util.GetLbEndpoints(endpointSlices, svcPort, family) - // Reconcile OVN, update the load balancer with current endpoints - - var currentLB string - // If any of the lbEps contain a host IP we add to worker/GR LB separately, and not to cluster LB - if hasHostEndpoints(eps.IPs) && config.Gateway.Mode == config.GatewayModeShared { - currentLB = loadbalancer.NodeLoadBalancer - if err := createPerNodeVIPs([]string{ip}, svcPort.Protocol, svcPort.Port, eps.IPs, eps.Port); err != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToUpdateOVNLoadBalancer", - "Error trying to update OVN LoadBalancer for Service %s/%s: %v", name, namespace, err) - return err - } - if c.serviceTracker.getLoadBalancer(name, namespace, vip) != currentLB { - txn := util.NewNBTxn() - // Need to ensure that if vip exists on cluster LB we remove it - // This can happen if endpoints originally had cluster only ips but now have host ips - if err := loadbalancer.DeleteLoadBalancerVIP(txn, clusterLB, vip); err != nil { - return err - } - if stdout, stderr, err := txn.Commit(); err != nil { - klog.Errorf("Error deleting VIP %s on OVN LoadBalancer %s", vip, clusterLB) - return fmt.Errorf("error deleting load balancer vip %v for %v"+ - "stdout: %q, stderr: %q, error: %v", - vip, clusterLB, stdout, stderr, err) - } - } - } else { - currentLB = clusterLB - if err = loadbalancer.CreateLoadBalancerVIPs(clusterLB, []string{ip}, svcPort.Port, eps.IPs, eps.Port); err != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToUpdateOVNLoadBalancer", - "Error trying to update OVN LoadBalancer for Service %s/%s: %v", name, namespace, err) - return err - } - if c.serviceTracker.getLoadBalancer(name, namespace, virtualIPKey(vip, svcPort.Protocol)) != currentLB { - // Need to ensure if this vip exists in the worker LBs that we remove it - // This can happen if the endpoints originally had host eps but now have cluster only ips - if err := deleteNodeVIPs([]string{ip}, svcPort.Protocol, svcPort.Port); err != nil { - klog.Errorf("Error deleting VIP %s on per node load balancers, error: %v", vip, err) - return err - } - } - } - // update the tracker with the VIP - c.serviceTracker.updateService(name, namespace, vip, svcPort.Protocol, currentLB) - // mark the vip as processed - vipsTracked.Delete(virtualIPKey(vip, svcPort.Protocol)) - - // Node Port - if svcPort.NodePort != 0 { - if err := createPerNodePhysicalVIPs(utilnet.IsIPv6String(ip), svcPort.Protocol, svcPort.NodePort, - eps.IPs, eps.Port); err != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToUpdateOVNLoadBalancer", - "Error trying to update OVN LoadBalancer for Service %s/%s: %v", - name, namespace, err) - return err - } - nodeIPs, err := getNodeIPs(utilnet.IsIPv6String(ip)) - if err != nil { - return err - } - for _, nodeIP := range nodeIPs { - vip := util.JoinHostPortInt32(nodeIP, svcPort.NodePort) - c.serviceTracker.updateService(name, namespace, vip, svcPort.Protocol, loadbalancer.NodeLoadBalancer) - // mark the vip as processed - vipsTracked.Delete(virtualIPKey(vip, svcPort.Protocol)) - } - } - - // Services ExternalIPs and LoadBalancer.IngressIPs have the same behavior in OVN - // so they are aggregated in a slice and processed together. - var externalIPs []string - // ExternalIP - for _, extIP := range service.Spec.ExternalIPs { - // only use the IPs of the same ClusterIP family - if utilnet.IsIPv6String(extIP) == utilnet.IsIPv6String(ip) { - externalIPs = append(externalIPs, extIP) - } - } - // LoadBalancer - for _, ingress := range service.Status.LoadBalancer.Ingress { - // only use the IPs of the same ClusterIP family - if ingress.IP != "" && utilnet.IsIPv6String(ingress.IP) == utilnet.IsIPv6String(ip) { - externalIPs = append(externalIPs, ingress.IP) - } - } - - // reconcile external IPs - if len(externalIPs) > 0 { - if err := createPerNodeVIPs(externalIPs, svcPort.Protocol, svcPort.Port, eps.IPs, eps.Port); err != nil { - klog.Errorf("Error in creating ExternalIP/IngressIP for svc %s, target port: %d - %v\n", name, eps.Port, err) - } - for _, extIP := range externalIPs { - vip := util.JoinHostPortInt32(extIP, svcPort.Port) - c.serviceTracker.updateService(name, namespace, vip, svcPort.Protocol, loadbalancer.NodeLoadBalancer) - // mark the vip as processed - vipsTracked.Delete(virtualIPKey(vip, svcPort.Protocol)) - } - } + if !c.repair.legacyLBsDeleted() { + if err := deleteServiceFromLegacyLBs(service); err != nil { + klog.Warningf("Failed to delete legacy vips for service %s: %v", key) + // Continue anyways, because once all services are synced, we'll delete + // the legacy load balancers } } - // at this point we have processed all vips we've found in the service - // so the remaining ones that we had in the vipsTracked variable should be deleted - // We remove them from OVN and from the tracker - err = deleteVIPsFromAllOVNBalancers(vipsTracked, name, namespace) - if err != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToDeleteOVNLoadBalancer", - "Error trying to delete the OVN LoadBalancer for Service %s/%s: %v", name, namespace, err) - return err - } - c.serviceTracker.deleteServiceVIPs(name, namespace, vipsTracked) - return nil -} - -func (c *Controller) addServiceToIdlingBalancer(vips sets.String, service *v1.Service) error { - for _, vipProtocol := range vips.List() { - vip, protocol := splitVirtualIPKey(vipProtocol) - lb, err := loadbalancer.GetOVNKubeIdlingLoadBalancer(protocol) - if err != nil { - return errors.Wrapf(err, "Error getting OVN idling LoadBalancer for protocol %s", protocol) - } - err = loadbalancer.UpdateLoadBalancer(lb, vip, []string{}) - if err != nil { - return errors.Wrapf(err, "Failed to update idling loadbalancer") - } - // update the tracker with the VIP - c.serviceTracker.updateService(service.Name, service.Namespace, vip, protocol, loadbalancer.IdlingLoadBalancer) - } + c.repair.serviceSynced(key) return nil } // RequestFullSync re-syncs every service that currently exists -func (c *Controller) RequestFullSync() error { +func (c *Controller) RequestFullSync() { klog.Info("Full service sync requested") services, err := c.serviceLister.List(labels.Everything()) if err != nil { - return err + klog.Errorf("Cached lister failed!? %v", err) + return } for _, service := range services { c.onServiceAdd(service) } - return nil } // handlers @@ -606,3 +457,12 @@ func serviceControllerKey(endpointSlice *discovery.EndpointSlice) (string, error } return fmt.Sprintf("%s/%s", endpointSlice.Namespace, serviceName), nil } + +// newRateLimiter makes a queue rate limiter. This limits re-queues somewhat more significantly than base qps. +// the client-go default qps is 10, but this is low for our level of scale. +func newRatelimiter(qps int) workqueue.RateLimiter { + return workqueue.NewMaxOfRateLimiter( + workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 1000*time.Second), + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(qps), qps*5)}, + ) +} diff --git a/go-controller/pkg/ovn/controller/services/services_controller_test.go b/go-controller/pkg/ovn/controller/services/services_controller_test.go index 336d554244..eaa1b94045 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller_test.go +++ b/go-controller/pkg/ovn/controller/services/services_controller_test.go @@ -5,7 +5,8 @@ import ( "net" "testing" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + globalconfig "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + ovnlb "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -20,15 +21,6 @@ import ( utilpointer "k8s.io/utils/pointer" ) -const ( - loadbalancerTCP = "a08ea426-2288-11eb-a30b-a8a1590cda29" - portGroupUUID = "58a1ef18-3649-11eb-bd94-a8a1590cda29" - gatewayRouter1 = "2e290f10-3652-11eb-839b-a8a1590cda29" - logicalSwitch1 = "17bde5e8-3652-11eb-b53b-a8a1590cda29" - idlingloadbalancerTCP = "a08ea426-2288-11eb-a30b-a8a1590cda30" - clusterPortGroupUUID = "a08ea426-2288-11eb-a30b-a8a1590cda31" -) - var alwaysReady = func() bool { return true } var FakeGRs = "GR_1 GR_2" @@ -55,7 +47,7 @@ func newControllerWithDBSetup(dbSetup libovsdbtest.TestSetup) (*serviceControlle nbClient, informerFactory.Core().V1().Services(), informerFactory.Discovery().V1beta1().EndpointSlices(), - clusterPortGroupUUID, + informerFactory.Core().V1().Nodes(), ) controller.servicesSynced = alwaysReady controller.endpointSlicesSynced = alwaysReady @@ -71,22 +63,49 @@ func (c *serviceController) close() { close(c.stopChan) } +// TestSyncServices - an end-to-end test for the services controller. func TestSyncServices(t *testing.T) { ns := "testns" serviceName := "foo" - config.Kubernetes.OVNEmptyLbEvents = true - config.IPv4Mode = true + + oldGateway := globalconfig.Gateway.Mode + oldClusterSubnet := globalconfig.Default.ClusterSubnets + globalconfig.Kubernetes.OVNEmptyLbEvents = true + globalconfig.IPv4Mode = true defer func() { - config.Kubernetes.OVNEmptyLbEvents = false - config.IPv4Mode = false + globalconfig.Kubernetes.OVNEmptyLbEvents = false + globalconfig.IPv4Mode = false + globalconfig.Gateway.Mode = oldGateway + globalconfig.Default.ClusterSubnets = oldClusterSubnet }() + _, cidr4, _ := net.ParseCIDR("10.128.0.0/16") + _, cidr6, _ := net.ParseCIDR("fe00::/64") + globalconfig.Default.ClusterSubnets = []globalconfig.CIDRNetworkEntry{{cidr4, 26}, {cidr6, 26}} + + outport := int32(3456) + tcp := v1.ProtocolTCP + + defaultNodes := map[string]nodeInfo{ + "node-a": { + name: "node-a", + nodeIPs: []string{"10.0.0.1"}, + gatewayRouterName: "gr-node-a", + switchName: "switch-node-a", + }, + "node-b": { + name: "node-b", + nodeIPs: []string{"10.0.0.2"}, + gatewayRouterName: "gr-node-b", + switchName: "switch-node-b", + }, + } tests := []struct { - name string - slice *discovery.EndpointSlice - service *v1.Service - updateTracker bool - ovnCmd []ovntest.ExpectedCmd + name string + slice *discovery.EndpointSlice + service *v1.Service + ovnCmd []ovntest.ExpectedCmd + gatewayMode string }{ { @@ -115,40 +134,28 @@ func TestSyncServices(t *testing.T) { }}, }, }, - updateTracker: true, ovnCmd: []ovntest.ExpectedCmd{ { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingloadbalancerTCP, - }, - { - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips \"192.168.1.1:80\"", - Output: "", - }, - { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, + Cmd: `ovn-nbctl --timeout=15 --format=json --data=json --columns=name,_uuid,protocol,external_ids,vips find load_balancer`, + Output: `{"data": []}`, }, { - Cmd: "ovn-nbctl --timeout=15 set load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips:\"192.168.1.1:80\"=\"\"", - Output: "", + Cmd: `ovn-nbctl --timeout=15 --no-heading --format=csv --data=bare --columns=name,load_balancer find logical_switch`, }, { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: gatewayRouter1, + Cmd: `ovn-nbctl --timeout=15 --no-heading --format=csv --data=bare --columns=name,load_balancer find logical_router`, }, { - Cmd: fmt.Sprintf("ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=%s", gatewayRouter1), - Output: "load_balancer_1", + Cmd: `ovn-nbctl --timeout=15 create load_balancer external_ids:k8s.ovn.org/kind=Service external_ids:k8s.ovn.org/owner=testns/foo name=Service_testns/foo_TCP_cluster options:event=false options:reject=true options:skip_snat=false protocol=tcp selection_fields=[] vips={"192.168.1.1:80"=""}`, + Output: "uuid-1", }, { - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer load_balancer_1 vips \"192.168.1.1:80\"", - Output: "", + Cmd: `ovn-nbctl --timeout=15 --may-exist ls-lb-add switch-node-a uuid-1 -- --may-exist ls-lb-add switch-node-b uuid-1 -- --may-exist lr-lb-add gr-node-a uuid-1 -- --may-exist lr-lb-add gr-node-b uuid-1`, }, }, }, { - name: "create OVN LoadBalancer from Single Stack NodePort Service without endpoints", + name: "update service without endpoints", slice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName + "ab23", @@ -170,100 +177,39 @@ func TestSyncServices(t *testing.T) { Port: 80, Protocol: v1.ProtocolTCP, TargetPort: intstr.FromInt(3456), - NodePort: 32766, }}, }, }, - updateTracker: true, ovnCmd: []ovntest.ExpectedCmd{ { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: gatewayRouter1, - }, - { - Cmd: `ovn-nbctl --timeout=15 get logical_router 2e290f10-3652-11eb-839b-a8a1590cda29 external_ids:physical_ips`, - Output: "5.5.5.5", - }, - { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: gatewayRouter1, - }, - { - Cmd: `ovn-nbctl --timeout=15 get logical_router 2e290f10-3652-11eb-839b-a8a1590cda29 external_ids:physical_ips`, - Output: "5.5.5.5", - }, - { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingloadbalancerTCP, + Cmd: `ovn-nbctl --timeout=15 --format=json --data=json --columns=name,_uuid,protocol,external_ids,vips find load_balancer`, + Output: `{"data":[["Service_testns/foo_TCP_cluster",["uuid","uuid-1"],"tcp",["map",[["k8s.ovn.org/kind","Service"],["k8s.ovn.org/owner","testns/foo"]]],["map",[["192.168.0.1:6443",""]]]]]}`, }, { - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips "192.168.1.1:80"` + - ` -- --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips "5.5.5.5:32766"`, - Output: "", + Cmd: `ovn-nbctl --timeout=15 --no-heading --format=csv --data=bare --columns=name,load_balancer find logical_switch`, + Output: `wrong-switch,uuid-1`, }, { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, + Cmd: `ovn-nbctl --timeout=15 --no-heading --format=csv --data=bare --columns=name,load_balancer find logical_router`, + Output: "gr-node-a,uuid-1\ngr-node-c,uuid-1", }, { - Cmd: "ovn-nbctl --timeout=15 set load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips:\"192.168.1.1:80\"=\"\"", - Output: "", - }, - { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: gatewayRouter1, - }, - { - Cmd: fmt.Sprintf("ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=%s", gatewayRouter1), - Output: "load_balancer_1", - }, - { - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer load_balancer_1 vips \"192.168.1.1:80\"", - Output: "", - }, - { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: gatewayRouter1, - }, - { - Cmd: fmt.Sprintf("ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=%s", gatewayRouter1), - Output: "load_balancer_1", - }, - { - Cmd: `ovn-nbctl --timeout=15 get logical_router 2e290f10-3652-11eb-839b-a8a1590cda29 external_ids:physical_ips`, - Output: "5.5.5.5", - }, - { - Cmd: "ovn-nbctl --timeout=15 set load_balancer load_balancer_1 vips:\"5.5.5.5:32766\"=\"\"", - Output: "", + Cmd: `ovn-nbctl --timeout=15 set load_balancer uuid-1 external_ids:k8s.ovn.org/kind=Service external_ids:k8s.ovn.org/owner=testns/foo name=Service_testns/foo_TCP_cluster options:event=false options:reject=true options:skip_snat=false protocol=tcp selection_fields=[] vips={"192.168.1.1:80"=""} ` + + `-- --may-exist ls-lb-add switch-node-a uuid-1 -- --may-exist ls-lb-add switch-node-b uuid-1 -- --if-exists ls-lb-del wrong-switch uuid-1 -- --may-exist lr-lb-add gr-node-b uuid-1 -- --if-exists lr-lb-del gr-node-c uuid-1`, }, }, }, { - name: "create OVN LoadBalancer from Single Stack Service with endpoints", + name: "remove service from legacy load balancers", slice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName + "ab23", Namespace: ns, Labels: map[string]string{discovery.LabelServiceName: serviceName}, }, - Ports: []discovery.EndpointPort{ - { - Name: utilpointer.StringPtr("tcp-example"), - Protocol: protoPtr(v1.ProtocolTCP), - Port: utilpointer.Int32Ptr(int32(3456)), - }, - }, + Ports: []discovery.EndpointPort{}, AddressType: discovery.AddressTypeIPv4, - Endpoints: []discovery.Endpoint{ - { - Conditions: discovery.EndpointConditions{ - Ready: utilpointer.BoolPtr(true), - }, - Addresses: []string{"10.0.0.2"}, - Topology: map[string]string{"kubernetes.io/hostname": "node-1"}, - }, - }, + Endpoints: []discovery.Endpoint{}, }, service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, @@ -279,55 +225,39 @@ func TestSyncServices(t *testing.T) { }}, }, }, - updateTracker: false, ovnCmd: []ovntest.ExpectedCmd{ { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingloadbalancerTCP, + Cmd: `ovn-nbctl --timeout=15 --format=json --data=json --columns=name,_uuid,protocol,external_ids,vips find load_balancer`, + Output: `{"data":[["Service_testns/foo_TCP_cluster",["uuid","uuid-1"],"tcp",["map",[["k8s.ovn.org/kind","Service"],["k8s.ovn.org/owner","testns/foo"]]],["map",[["192.168.0.1:6443",""]]]],["",["uuid","uuid-legacy"],"tcp",["map",[["TCP_lb_gateway_router",""]]],["map",[["192.168.1.1:80",""]]]]]}`, }, { - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips \"192.168.1.1:80\"", - Output: "", + Cmd: `ovn-nbctl --timeout=15 --no-heading --format=csv --data=bare --columns=name,load_balancer find logical_switch`, + Output: "switch-node-a,uuid-1\nswitch-node-b,uuid-1", }, { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, + Cmd: `ovn-nbctl --timeout=15 --no-heading --format=csv --data=bare --columns=name,load_balancer find logical_router`, + Output: "gr-node-a,uuid-1\ngr-node-b,uuid-1", }, { - Cmd: `ovn-nbctl --timeout=15 set load_balancer ` + loadbalancerTCP + ` vips:"192.168.1.1:80"="10.0.0.2:3456"`, - Output: "", + Cmd: `ovn-nbctl --timeout=15 set load_balancer uuid-1 external_ids:k8s.ovn.org/kind=Service external_ids:k8s.ovn.org/owner=testns/foo name=Service_testns/foo_TCP_cluster options:event=false options:reject=true options:skip_snat=false protocol=tcp selection_fields=[] vips={"192.168.1.1:80"=""}`, }, { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: FakeGRs, - }, - { - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_1`, - Output: "load_balancer_1", - }, - { - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer load_balancer_1 vips "192.168.1.1:80"`, - Output: "", - }, - { - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_2`, - Output: "", + Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer uuid-legacy vips "192.168.1.1:80"`, }, }, }, { - name: "create OVN LoadBalancer from Dual Stack Service with dual stack endpoints", + name: "transition to endpoints, create nodeport", slice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceName + "ab23", + Name: serviceName + "ab1", Namespace: ns, Labels: map[string]string{discovery.LabelServiceName: serviceName}, }, Ports: []discovery.EndpointPort{ { - Name: utilpointer.StringPtr("tcp-example"), - Protocol: protoPtr(v1.ProtocolTCP), - Port: utilpointer.Int32Ptr(int32(3456)), + Protocol: &tcp, + Port: &outport, }, }, AddressType: discovery.AddressTypeIPv4, @@ -336,8 +266,7 @@ func TestSyncServices(t *testing.T) { Conditions: discovery.EndpointConditions{ Ready: utilpointer.BoolPtr(true), }, - Addresses: []string{"10.0.0.2"}, - Topology: map[string]string{"kubernetes.io/hostname": "node-1"}, + Addresses: []string{"10.128.0.2", "10.128.1.2"}, }, }, }, @@ -352,49 +281,49 @@ func TestSyncServices(t *testing.T) { Port: 80, Protocol: v1.ProtocolTCP, TargetPort: intstr.FromInt(3456), + NodePort: 8989, }}, }, }, - updateTracker: false, ovnCmd: []ovntest.ExpectedCmd{ { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingloadbalancerTCP, - }, - { - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips \"192.168.1.1:80\"", - Output: "", + Cmd: `ovn-nbctl --timeout=15 --format=json --data=json --columns=name,_uuid,protocol,external_ids,vips find load_balancer`, + Output: `{"data":[["Service_testns/foo_TCP_cluster",["uuid","uuid-1"],"tcp",["map",[["k8s.ovn.org/kind","Service"],["k8s.ovn.org/owner","testns/foo"]]],["map",[["192.168.0.1:6443",""]]]]]}`, }, { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, + Cmd: `ovn-nbctl --timeout=15 --no-heading --format=csv --data=bare --columns=name,load_balancer find logical_switch`, + Output: "switch-node-a,uuid-1\nswitch-node-b,uuid-1", }, { - Cmd: `ovn-nbctl --timeout=15 set load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips:"192.168.1.1:80"="10.0.0.2:3456"`, - Output: "", + Cmd: `ovn-nbctl --timeout=15 --no-heading --format=csv --data=bare --columns=name,load_balancer find logical_router`, + Output: "gr-node-a,uuid-1\ngr-node-b,uuid-1", }, { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: FakeGRs, + Cmd: `ovn-nbctl --timeout=15 create load_balancer external_ids:k8s.ovn.org/kind=Service external_ids:k8s.ovn.org/owner=testns/foo name=Service_testns/foo_TCP_node_router+switch_node-a options:event=false options:reject=true options:skip_snat=false protocol=tcp selection_fields=[] vips={"10.0.0.1:8989"="10.128.0.2:3456,10.128.1.2:3456"}`, + Output: "uuid-rs-nodea", }, { - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_1`, - Output: "load_balancer_1", + Cmd: `ovn-nbctl --timeout=15 create load_balancer external_ids:k8s.ovn.org/kind=Service external_ids:k8s.ovn.org/owner=testns/foo name=Service_testns/foo_TCP_node_router+switch_node-b options:event=false options:reject=true options:skip_snat=false protocol=tcp selection_fields=[] vips={"10.0.0.2:8989"="10.128.0.2:3456,10.128.1.2:3456"}`, + Output: "uuid-rs-nodeb", }, { - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer load_balancer_1 vips "192.168.1.1:80"`, - Output: "", - }, - { - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_2`, - Output: "", + Cmd: `ovn-nbctl --timeout=15 set load_balancer uuid-1 external_ids:k8s.ovn.org/kind=Service external_ids:k8s.ovn.org/owner=testns/foo name=Service_testns/foo_TCP_cluster options:event=false options:reject=true options:skip_snat=false protocol=tcp selection_fields=[] vips={"192.168.1.1:80"="10.128.0.2:3456,10.128.1.2:3456"}` + + ` -- --may-exist ls-lb-add switch-node-a uuid-rs-nodea -- --may-exist lr-lb-add gr-node-a uuid-rs-nodea` + + ` -- --may-exist ls-lb-add switch-node-b uuid-rs-nodeb -- --may-exist lr-lb-add gr-node-b uuid-rs-nodeb`, }, }, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + for i, tt := range tests { + t.Run(fmt.Sprintf("%d_%s", i, tt.name), func(t *testing.T) { + if tt.gatewayMode != "" { + globalconfig.Gateway.Mode = globalconfig.GatewayMode(tt.gatewayMode) + } else { + globalconfig.Gateway.Mode = globalconfig.GatewayModeShared + } + + ovnlb.TestOnlySetCache(nil) controller, err := newController() if err != nil { t.Fatalf("Error creating controller: %v", err) @@ -403,10 +332,8 @@ func TestSyncServices(t *testing.T) { // Add objects to the Store controller.endpointSliceStore.Add(tt.slice) controller.serviceStore.Add(tt.service) - if tt.updateTracker { - controller.serviceTracker.updateKubernetesService(tt.service, "") - } + controller.nodeTracker.nodes = defaultNodes // Expected OVN commands fexec := ovntest.NewLooseCompareFakeExec() for _, cmd := range tt.ovnCmd { @@ -417,7 +344,7 @@ func TestSyncServices(t *testing.T) { if err != nil { t.Errorf("fexec error: %v", err) } - err = controller.syncServices(ns + "/" + serviceName) + err = controller.syncService(ns + "/" + serviceName) if err != nil { t.Errorf("syncServices error: %v", err) } @@ -428,524 +355,3 @@ func TestSyncServices(t *testing.T) { }) } } - -// A service can mutate its ports, we need to be sure we don´t left dangling ports -func TestUpdateServicePorts(t *testing.T) { - config.Kubernetes.OVNEmptyLbEvents = true - defer func() { - config.Kubernetes.OVNEmptyLbEvents = false - }() - - // Expected OVN commands - fexec := ovntest.NewFakeExec() - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingloadbalancerTCP, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips \"192.168.1.1:80\"", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, - }) - // Add a new loadbalancer with the Service Port 80 - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 set load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips:"192.168.1.1:80"="10.0.0.2:3456"`, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: FakeGRs, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_1`, - Output: "load_balancer_1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_2`, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer load_balancer_1 vips "192.168.1.1:80"`, - Output: "", - }) - // update service starts here - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingloadbalancerTCP, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips \"192.168.1.1:8888\"", - Output: idlingloadbalancerTCP, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, - }) - // Add a new loadbalancer with the new Service Port 8888 - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 set load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips:"192.168.1.1:8888"="10.0.0.2:3456"`, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: FakeGRs, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_1`, - Output: "load_balancer_1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_2`, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer load_balancer_1 vips "192.168.1.1:8888"`, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: gatewayRouter1, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, - }) - // Remove the old ServicePort - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: fmt.Sprintf("ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=%s", gatewayRouter1), - Output: "load_balancer_1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: fmt.Sprintf("ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-tcp=2e290f10-3652-11eb-839b-a8a1590cda29"), - Output: "node_load_balancer_1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips "192.168.1.1:80"` + - ` -- --if-exists remove load_balancer load_balancer_1 vips "192.168.1.1:80"` + - ` -- --if-exists remove load_balancer node_load_balancer_1 vips "192.168.1.1:80"`, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingloadbalancerTCP, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips \"192.168.1.1:80\"", - Output: "", - }) - - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - - ns := "testns" - serviceName := "foo" - slice := &discovery.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName + "ab23", - Namespace: ns, - Labels: map[string]string{discovery.LabelServiceName: serviceName}, - }, - Ports: []discovery.EndpointPort{ - { - Name: utilpointer.StringPtr("tcp-example"), - Protocol: protoPtr(v1.ProtocolTCP), - Port: utilpointer.Int32Ptr(int32(3456)), - }, - }, - AddressType: discovery.AddressTypeIPv4, - Endpoints: []discovery.Endpoint{ - { - Conditions: discovery.EndpointConditions{ - Ready: utilpointer.BoolPtr(true), - }, - Addresses: []string{"10.0.0.2"}, - Topology: map[string]string{"kubernetes.io/hostname": "node-1"}, - }, - }, - } - service := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeClusterIP, - ClusterIP: "192.168.1.1", - ClusterIPs: []string{"192.168.1.1"}, - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{{ - Port: 80, - Protocol: v1.ProtocolTCP, - TargetPort: intstr.FromInt(3456), - }}, - }, - } - controller, err := newController() - if err != nil { - t.Fatalf("Error creating controller: %v", err) - } - defer controller.close() - // Process the first service - controller.endpointSliceStore.Add(slice) - controller.serviceStore.Add(service) - controller.syncServices(ns + "/" + serviceName) - - // Update the service with a different Port - serviceNew := service.DeepCopy() - serviceNew.Spec.Ports[0].Port = 8888 - controller.serviceStore.Delete(service) - controller.serviceStore.Add(serviceNew) - // sync service - controller.syncServices(ns + "/" + serviceName) - if controller.serviceTracker.hasServiceVIP(serviceName, ns, "192.168.1.1:80", v1.ProtocolTCP) { - t.Fatalf("Service with port 80 should not exist") - } - if !controller.serviceTracker.hasServiceVIP(serviceName, ns, "192.168.1.1:8888", v1.ProtocolTCP) { - t.Fatalf("Service with port 8888 should exist") - } - if !fexec.CalledMatchesExpected() { - t.Error(fexec.ErrorDesc()) - } -} - -// A service can mutate its ports, we need to be sure we don´t left dangling ports -func TestUpdateServiceEndpointsToHost(t *testing.T) { - config.Kubernetes.OVNEmptyLbEvents = true - defer func() { - config.Kubernetes.OVNEmptyLbEvents = false - }() - - // Expected OVN commands - fexec := ovntest.NewFakeExec() - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingloadbalancerTCP, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips \"192.168.1.1:80\"", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, - }) - // Add a new loadbalancer with the Service Port 80 - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 set load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips:"192.168.1.1:80"="10.128.0.2:3456"`, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: FakeGRs, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_1`, - Output: "load_balancer_1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-tcp=1`, - Output: "load_balancer_worker_1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_2`, - Output: "load_balancer_2", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-tcp=2`, - Output: "load_balancer_worker_2", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer load_balancer_1 vips "192.168.1.1:80"` + - ` -- --if-exists remove load_balancer load_balancer_worker_1 vips "192.168.1.1:80"` + - ` -- --if-exists remove load_balancer load_balancer_2 vips "192.168.1.1:80"` + - ` -- --if-exists remove load_balancer load_balancer_worker_2 vips "192.168.1.1:80"`, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, - }) - // Update endpoints to have host endpoint in shared gw mode - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: FakeGRs, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_1`, - Output: "load_balancer_1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 get logical_router GR_1 external_ids:physical_ips`, - Output: "2.2.2.2", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-tcp=1`, - Output: "load_balancer_worker_1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_2`, - Output: "load_balancer_2", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 get logical_router GR_2 external_ids:physical_ips`, - Output: "2.2.2.3", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-tcp=2`, - Output: "load_balancer_worker_2", - }) - // endpoint is self node IP, so need to use special masquerade endpoint - // use regular backend on the worker switch LB - // adding to second node will not use special masquerade - // and regular endpoint IP on the 2nd worker switch - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 set load_balancer load_balancer_1 vips:"192.168.1.1:80"="169.254.169.2:3456"` + - ` -- set load_balancer load_balancer_worker_1 vips:"192.168.1.1:80"="2.2.2.2:3456"` + - ` -- set load_balancer load_balancer_2 vips:"192.168.1.1:80"="2.2.2.2:3456"` + - ` -- set load_balancer load_balancer_worker_2 vips:"192.168.1.1:80"="2.2.2.2:3456"`, - Output: "", - }) - // Ensure the VIP entry is removed on the cluster wide TCP load balancer - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips "192.168.1.1:80"`, - Output: "", - }) - - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - - ns := "testns" - serviceName := "foo" - slice := &discovery.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName + "ab23", - Namespace: ns, - Labels: map[string]string{discovery.LabelServiceName: serviceName}, - }, - Ports: []discovery.EndpointPort{ - { - Name: utilpointer.StringPtr("tcp-example"), - Protocol: protoPtr(v1.ProtocolTCP), - Port: utilpointer.Int32Ptr(int32(3456)), - }, - }, - AddressType: discovery.AddressTypeIPv4, - Endpoints: []discovery.Endpoint{ - { - Conditions: discovery.EndpointConditions{ - Ready: utilpointer.BoolPtr(true), - }, - Addresses: []string{"10.128.0.2"}, - Topology: map[string]string{"kubernetes.io/hostname": "node-1"}, - }, - }, - } - service := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeClusterIP, - ClusterIP: "192.168.1.1", - ClusterIPs: []string{"192.168.1.1"}, - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{{ - Port: 80, - Protocol: v1.ProtocolTCP, - TargetPort: intstr.FromInt(3456), - }}, - }, - } - oldClusterSubnet := config.Default.ClusterSubnets - oldGwMode := config.Gateway.Mode - defer func() { - config.Gateway.Mode = oldGwMode - config.Default.ClusterSubnets = oldClusterSubnet - }() - _, cidr, _ := net.ParseCIDR("10.128.0.0/24") - config.Default.ClusterSubnets = []config.CIDRNetworkEntry{{cidr, 26}} - config.Gateway.Mode = config.GatewayModeShared - controller, err := newController() - if err != nil { - t.Fatalf("Error creating controller: %v", err) - } - defer controller.close() - // Process the first service - controller.endpointSliceStore.Add(slice) - controller.serviceStore.Add(service) - controller.syncServices(ns + "/" + serviceName) - - // Update endpoints with host network pod - epsNew := slice.DeepCopy() - epsNew.Endpoints = []discovery.Endpoint{ - { - Conditions: discovery.EndpointConditions{ - Ready: utilpointer.BoolPtr(true), - }, - Addresses: []string{"2.2.2.2"}, - Topology: map[string]string{"kubernetes.io/hostname": "node-1"}, - }} - controller.endpointSliceStore.Delete(slice) - controller.endpointSliceStore.Add(epsNew) - // sync service - controller.syncServices(ns + "/" + serviceName) - - if !fexec.CalledMatchesExpected() { - t.Error(fexec.ErrorDesc()) - } -} - -// Update a service that was not idled, change endpoints that are both non host network and ensure that -// there are no unnecessary remove cmds -func TestUpdateServiceEndpointsLessRemoveOps(t *testing.T) { - config.Kubernetes.OVNEmptyLbEvents = true - defer func() { - config.Kubernetes.OVNEmptyLbEvents = false - }() - // Expected OVN commands - fexec := ovntest.NewFakeExec() - // First sync we expect the redundant remove commands - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingloadbalancerTCP, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips \"192.168.1.1:80\"", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, - }) - // Add a new loadbalancer with the Service Port 80 - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 set load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips:"192.168.1.1:80"="10.128.0.2:3456"`, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: FakeGRs, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_1`, - Output: "load_balancer_1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-tcp=1`, - Output: "load_balancer_worker_1", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_2`, - Output: "load_balancer_2", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-tcp=2`, - Output: "load_balancer_worker_2", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer load_balancer_1 vips "192.168.1.1:80"` + - ` -- --if-exists remove load_balancer load_balancer_worker_1 vips "192.168.1.1:80"` + - ` -- --if-exists remove load_balancer load_balancer_2 vips "192.168.1.1:80"` + - ` -- --if-exists remove load_balancer load_balancer_worker_2 vips "192.168.1.1:80"`, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, - }) - // Update endpoints to have new endpoint in shared gw mode, should not call redundant remove ops on idling, worker lbs - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 set load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips:"192.168.1.1:80"="10.128.0.6:3456"`, - Output: "", - }) - - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - - ns := "testns" - serviceName := "foo" - slice := &discovery.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName + "ab23", - Namespace: ns, - Labels: map[string]string{discovery.LabelServiceName: serviceName}, - }, - Ports: []discovery.EndpointPort{ - { - Name: utilpointer.StringPtr("tcp-example"), - Protocol: protoPtr(v1.ProtocolTCP), - Port: utilpointer.Int32Ptr(int32(3456)), - }, - }, - AddressType: discovery.AddressTypeIPv4, - Endpoints: []discovery.Endpoint{ - { - Conditions: discovery.EndpointConditions{ - Ready: utilpointer.BoolPtr(true), - }, - Addresses: []string{"10.128.0.2"}, - Topology: map[string]string{"kubernetes.io/hostname": "node-1"}, - }, - }, - } - service := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: ns}, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeClusterIP, - ClusterIP: "192.168.1.1", - ClusterIPs: []string{"192.168.1.1"}, - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{{ - Port: 80, - Protocol: v1.ProtocolTCP, - TargetPort: intstr.FromInt(3456), - }}, - }, - } - oldClusterSubnet := config.Default.ClusterSubnets - oldGwMode := config.Gateway.Mode - defer func() { - config.Gateway.Mode = oldGwMode - config.Default.ClusterSubnets = oldClusterSubnet - }() - _, cidr, _ := net.ParseCIDR("10.128.0.0/24") - config.Default.ClusterSubnets = []config.CIDRNetworkEntry{{cidr, 26}} - config.Gateway.Mode = config.GatewayModeShared - controller, err := newController() - if err != nil { - t.Fatalf("Error creating controller: %v", err) - } - defer controller.close() - // Process the first service - controller.endpointSliceStore.Add(slice) - controller.serviceStore.Add(service) - controller.syncServices(ns + "/" + serviceName) - - // Update endpoints with host network pod - epsNew := slice.DeepCopy() - epsNew.Endpoints = []discovery.Endpoint{ - { - Conditions: discovery.EndpointConditions{ - Ready: utilpointer.BoolPtr(true), - }, - Addresses: []string{"10.128.0.6"}, - Topology: map[string]string{"kubernetes.io/hostname": "node-1"}, - }} - controller.endpointSliceStore.Delete(slice) - controller.endpointSliceStore.Add(epsNew) - // sync service - controller.syncServices(ns + "/" + serviceName) - - if !fexec.CalledMatchesExpected() { - t.Error(fexec.ErrorDesc()) - } -} - -// protoPtr takes a Protocol and returns a pointer to it. -func protoPtr(proto v1.Protocol) *v1.Protocol { - return &proto -} diff --git a/go-controller/pkg/ovn/controller/services/utils.go b/go-controller/pkg/ovn/controller/services/utils.go index 3cd1cda262..9fe39d4cb7 100644 --- a/go-controller/pkg/ovn/controller/services/utils.go +++ b/go-controller/pkg/ovn/controller/services/utils.go @@ -3,338 +3,116 @@ package services import ( "fmt" "net" + "regexp" "strings" - "github.com/pkg/errors" - - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/gateway" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + globalconfig "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + ovnlb "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" ) -func deleteVIPsFromAllOVNBalancers(vips sets.String, name, namespace string) error { - err := deleteVIPsFromNonIdlingOVNBalancers(vips, name, namespace) - if err != nil { - return errors.Wrapf(err, "Failed to delete vips from ovn balancers %s %s", name, namespace) - } - err = deleteVIPsFromIdlingBalancer(vips, name, namespace) - if err != nil { - return errors.Wrapf(err, "Failed to delete vips from idling balancers %s %s", name, namespace) - } - return nil -} +// deleteServiceFrom LegacyLBs removes any of a service's vips from +// the legacy shared load balancers. +// This misses cleaning up NodePort services, but those will be caught +// when the repair PostSync is done. +func deleteServiceFromLegacyLBs(service *v1.Service) error { + vipPortsPerProtocol := map[v1.Protocol]sets.String{} -// deleteVIPsFromNonIdlingOVNBalancers removes the given vips for all the loadbalancers but -// the idling ones. This includes the cluster loadbalancer, the gateway routers loadbalancers -// and the node switch ones. -func deleteVIPsFromNonIdlingOVNBalancers(vips sets.String, name, namespace string) error { - if len(vips) == 0 { - return nil - } - // NodePort and ExternalIPs use loadbalancers in each node - gatewayRouters, _, err := gateway.GetOvnGateways() - if err != nil { - return errors.Wrapf(err, "failed to retrieve OVN gateway routers") + // Generate list of vip:port by proto + ips := append([]string{}, service.Spec.ClusterIPs...) + if len(ips) == 0 { + ips = append(ips, service.Spec.ClusterIP) } - - vipsPerProtocol := map[v1.Protocol]sets.String{} - lbsPerProtocol := map[v1.Protocol]sets.String{} - foundProtocols := map[v1.Protocol]struct{}{} - - // Get load balancers for each Node - for vipKey := range vips { - // the VIP is stored with the format IP:Port/Protocol - vip, proto := splitVirtualIPKey(vipKey) - foundProtocols[proto] = struct{}{} - if _, ok := vipsPerProtocol[proto]; !ok { - vipsPerProtocol[proto] = sets.NewString(vip) - } else { - vipsPerProtocol[proto].Insert(vip) - } - klog.Infof("Deleting VIP: %s from idling OVN LoadBalancer for service %s on namespace %s", - vip, name, namespace) - if _, ok := lbsPerProtocol[proto]; ok { - // already got the load balancers for this protocol, don't do it again - continue - } - lbID, err := loadbalancer.GetOVNKubeLoadBalancer(proto) - if err != nil { - klog.Errorf("Error getting OVN LoadBalancer for protocol %s", proto) - return err - } - lbsPerProtocol[proto] = sets.NewString(lbID) - for _, gatewayRouter := range gatewayRouters { - gatewayLB, err := gateway.GetGatewayLoadBalancer(gatewayRouter, proto) - if err != nil { - klog.Warningf("Service Sync: Gateway router %s does not have load balancer (%v)", - gatewayRouter, err) - // TODO: why continue? should we error and requeue and retry? - continue - } - lbsPerProtocol[proto].Insert(gatewayLB) - workerNode := util.GetWorkerFromGatewayRouter(gatewayRouter) - workerLB, err := loadbalancer.GetWorkerLoadBalancer(workerNode, proto) - if err != nil { - klog.Errorf("Worker switch %s does not have load balancer (%v)", workerNode, err) - continue - } - lbsPerProtocol[proto].Insert(workerLB) - } + ips = append(ips, service.Spec.ExternalIPs...) + for _, ingress := range service.Status.LoadBalancer.Ingress { + ips = append(ips, ingress.IP) } - - txn := util.NewNBTxn() - for proto := range foundProtocols { - if err := loadbalancer.DeleteLoadBalancerVIPs(txn, lbsPerProtocol[proto].List(), vipsPerProtocol[proto].List()); err != nil { - klog.Errorf("Error deleting VIP %v on OVN LoadBalancer %v: %v", vipsPerProtocol[proto].List(), lbsPerProtocol[proto].List(), err) - return err + for _, svcPort := range service.Spec.Ports { + proto := svcPort.Protocol + ipPorts := make([]string, 0, len(ips)) + for _, ip := range ips { + ipPorts = append(ipPorts, util.JoinHostPortInt32(ip, svcPort.Port)) } - } - if stdout, stderr, err := txn.Commit(); err != nil { - klog.Errorf("Error deleting VIPs %v on OVN LoadBalancers %v", vips.List(), lbsPerProtocol) - return fmt.Errorf("error deleting load balancer %v VIPs %v"+ - "stdout: %q, stderr: %q, error: %v", - lbsPerProtocol, vips.List(), stdout, stderr, err) - } - - return nil -} - -func deleteVIPsFromIdlingBalancer(vipProtocols sets.String, name, namespace string) error { - // The idling lb is enabled only when configured - if !config.Kubernetes.OVNEmptyLbEvents { - return nil - } - vipsPerProtocol := map[v1.Protocol]sets.String{} - lbsPerProtocol := map[v1.Protocol]sets.String{} - foundProtocols := map[v1.Protocol]struct{}{} - - // Obtain the VIPs associated to the Service - for vipKey := range vipProtocols { - // the VIP is stored with the format IP:Port/Protocol - vip, proto := splitVirtualIPKey(vipKey) - if _, ok := vipsPerProtocol[proto]; !ok { - vipsPerProtocol[proto] = sets.NewString(vip) + if _, ok := vipPortsPerProtocol[proto]; !ok { + vipPortsPerProtocol[proto] = sets.NewString(ipPorts...) } else { - vipsPerProtocol[proto].Insert(vip) - } - foundProtocols[proto] = struct{}{} - klog.Infof("Deleting VIP: %s from idling OVN LoadBalancer for service %s on namespace %s", - vip, name, namespace) - if _, ok := lbsPerProtocol[proto]; ok { - // lb already found for this protocol, don't do it again - continue - } - lbID, err := loadbalancer.GetOVNKubeIdlingLoadBalancer(proto) - if err != nil { - klog.Errorf("Error getting OVN idling LoadBalancer for protocol %s %v", proto, err) - return err - } - lbsPerProtocol[proto] = sets.NewString(lbID) - } - - txn := util.NewNBTxn() - for proto := range foundProtocols { - if err := loadbalancer.DeleteLoadBalancerVIPs(txn, lbsPerProtocol[proto].List(), vipsPerProtocol[proto].List()); err != nil { - klog.Errorf("Error deleting VIPs %v on idling OVN LoadBalancer %s %v", - vipsPerProtocol[proto].List(), lbsPerProtocol[proto].List(), err) - return err + vipPortsPerProtocol[proto].Insert(ipPorts...) } } - if stdout, stderr, err := txn.Commit(); err != nil { - klog.Errorf("Error deleting VIPs %v on idling OVN LoadBalancers %v", vipProtocols.List(), lbsPerProtocol) - return fmt.Errorf("error deleting idling load balancer %v VIPs %v"+ - "stdout: %q, stderr: %q, error: %v", - lbsPerProtocol, vipProtocols.List(), stdout, stderr, err) - } - - return nil -} -// createPerNodeVIPs adds load balancers on a per node basis for GR and worker switch LBs using service IPs -func createPerNodeVIPs(svcIPs []string, protocol v1.Protocol, sourcePort int32, targetIPs []string, targetPort int32) error { - if len(svcIPs) == 0 { - return fmt.Errorf("unable to create per node VIPs...no service IPs provided") - } - klog.V(5).Infof("Creating Node VIPs - %s, %d, [%v], %d", protocol, sourcePort, targetIPs, targetPort) - // Each gateway has a separate load-balancer for N/S traffic - gatewayRouters, _, err := gateway.GetOvnGateways() + legacyLBs, err := findLegacyLBs() if err != nil { return err } + if len(legacyLBs) == 0 { + return nil + } - lbConfig := make([]loadbalancer.Entry, 0, len(gatewayRouters)) - - for _, gatewayRouter := range gatewayRouters { - gatewayLB, err := gateway.GetGatewayLoadBalancer(gatewayRouter, protocol) - if err != nil { - klog.Errorf("Gateway router %s does not have load balancer (%v)", - gatewayRouter, err) - continue - } - physicalIPs, err := gateway.GetGatewayPhysicalIPs(gatewayRouter) - if err != nil { - klog.Errorf("Gateway router %s does not have physical ip (%v)", gatewayRouter, err) - continue - } - - var newTargets []string + toRemove := []ovnlb.DeleteVIPEntry{} - if config.Gateway.Mode == config.GatewayModeShared { - // If self ip is in target list, we need to use special IP to allow hairpin back to host - newTargets = util.UpdateIPsSlice(targetIPs, physicalIPs, []string{types.V4HostMasqueradeIP, types.V6HostMasqueradeIP}) - } else { - newTargets = targetIPs + // Find any legacy LBs with these vips + for _, lb := range legacyLBs { + r := ovnlb.DeleteVIPEntry{ + LBUUID: lb.UUID, } - lbConfig = append(lbConfig, loadbalancer.Entry{ - LoadBalancer: gatewayLB, - SourceIPS: svcIPs, - SourcePort: sourcePort, - TargetIPs: newTargets, - TargetPort: targetPort, - }) - if config.Gateway.Mode == config.GatewayModeShared { - workerNode := util.GetWorkerFromGatewayRouter(gatewayRouter) - workerLB, err := loadbalancer.GetWorkerLoadBalancer(workerNode, protocol) - if err != nil { - klog.Errorf("Worker switch %s does not have load balancer (%v)", workerNode, err) - return err + proto := v1.Protocol(strings.ToUpper(lb.Protocol)) + for _, vip := range vipPortsPerProtocol[proto].List() { + if _, ok := lb.VIPs[vip]; ok { + r.VIPs = append(r.VIPs, vip) } - lbConfig = append(lbConfig, loadbalancer.Entry{ - LoadBalancer: workerLB, - SourceIPS: svcIPs, - SourcePort: sourcePort, - TargetIPs: targetIPs, - TargetPort: targetPort, - }) } - } - return loadbalancer.BundleCreateLoadBalancerVIPs(lbConfig) -} -// createPerNodePhysicalVIPs adds load balancers on a per node basis for GR and worker switch LBs using physical IPs -func createPerNodePhysicalVIPs(isIPv6 bool, protocol v1.Protocol, sourcePort int32, targetIPs []string, targetPort int32) error { - klog.V(5).Infof("Creating Node VIPs - %s, %d, [%v], %d", protocol, sourcePort, targetIPs, targetPort) - // Each gateway has a separate load-balancer for N/S traffic - gatewayRouters, _, err := gateway.GetOvnGateways() - if err != nil { - return err - } - - lbConfig := make([]loadbalancer.Entry, 0, len(gatewayRouters)) - - for _, gatewayRouter := range gatewayRouters { - gatewayLB, err := gateway.GetGatewayLoadBalancer(gatewayRouter, protocol) - if err != nil { - klog.Errorf("Gateway router %s does not have load balancer (%v)", - gatewayRouter, err) - continue - } - physicalIPs, err := gateway.GetGatewayPhysicalIPs(gatewayRouter) - if err != nil { - klog.Errorf("Gateway router %s does not have physical ip (%v)", gatewayRouter, err) - continue + if len(r.VIPs) > 0 { + toRemove = append(toRemove, r) } - // Filter only phyiscal IPs of the same family - physicalIPs, err = util.MatchAllIPStringFamily(isIPv6, physicalIPs) - if err != nil { - klog.Errorf("Failed to find node physical IPs, for gateway: %s, error: %v", gatewayRouter, err) - return err - } - - var newTargets []string - - if config.Gateway.Mode == config.GatewayModeShared { - // If self ip is in target list, we need to use special IP to allow hairpin back to host - newTargets = util.UpdateIPsSlice(targetIPs, physicalIPs, []string{types.V4HostMasqueradeIP, types.V6HostMasqueradeIP}) - } else { - newTargets = targetIPs - } - - lbConfig = append(lbConfig, loadbalancer.Entry{ - LoadBalancer: gatewayLB, - SourceIPS: physicalIPs, - SourcePort: sourcePort, - TargetIPs: newTargets, - TargetPort: targetPort, - }) + } - if config.Gateway.Mode == config.GatewayModeShared { - workerNode := util.GetWorkerFromGatewayRouter(gatewayRouter) - workerLB, err := loadbalancer.GetWorkerLoadBalancer(workerNode, protocol) - if err != nil { - klog.Errorf("Worker switch %s does not have load balancer (%v)", workerNode, err) - return err - } - lbConfig = append(lbConfig, loadbalancer.Entry{ - LoadBalancer: workerLB, - SourceIPS: physicalIPs, - SourcePort: sourcePort, - TargetIPs: targetIPs, - TargetPort: targetPort, - }) - } + if err := ovnlb.DeleteLoadBalancerVIPs(toRemove); err != nil { + return fmt.Errorf("failed to delete vip(s) from legacy load balancers: %w", err) } - return loadbalancer.BundleCreateLoadBalancerVIPs(lbConfig) + return nil } -// deleteNodeVIPs removes load balancers on a per node basis for GR and worker switch LBs -func deleteNodeVIPs(svcIPs []string, protocol v1.Protocol, sourcePort int32) error { - klog.V(5).Infof("Searching to remove Gateway VIPs - %s, %d", protocol, sourcePort) - gatewayRouters, _, err := gateway.GetOvnGateways() +// getLegacyLBs returns all of the "legacy" non-per-Service load balancers +// This is any load balancer with one of the following external ID keys +// - k8s-worker-lb- +// - k8s-cluster-lb- +// - _lb_gateway_router +func findLegacyLBs() ([]ovnlb.CachedLB, error) { + lbCache, err := ovnlb.GetLBCache() if err != nil { - klog.Errorf("Error while searching for gateways: %v", err) - return err + return nil, err } - var loadBalancers []string - for _, gatewayRouter := range gatewayRouters { - gatewayLB, err := gateway.GetGatewayLoadBalancer(gatewayRouter, protocol) - if err != nil { - klog.Errorf("Gateway router %s does not have load balancer (%v)", gatewayRouter, err) + lbs := lbCache.Find(nil) + + legacyLBPattern := regexp.MustCompile(`(k8s-(worker|cluster)-lb-(tcp|udp|sctp)|(TCP|UDP|SCTP)_lb_gateway_router)`) + + out := []ovnlb.CachedLB{} + for _, lb := range lbs { + // legacy LBs had no name + if lb.Name != "" { continue } - loadBalancers = append(loadBalancers, gatewayLB) - if config.Gateway.Mode == config.GatewayModeShared { - workerNode := util.GetWorkerFromGatewayRouter(gatewayRouter) - workerLB, err := loadbalancer.GetWorkerLoadBalancer(workerNode, protocol) - if err != nil { - klog.Errorf("Worker switch %s does not have load balancer (%v)", workerNode, err) + for key := range lb.ExternalIDs { + if legacyLBPattern.MatchString(key) { + out = append(out, *lb) continue } - loadBalancers = append(loadBalancers, workerLB) } } - vips := make([]string, 0, len(svcIPs)) - for _, ip := range svcIPs { - vips = append(vips, util.JoinHostPortInt32(ip, sourcePort)) - } - - klog.V(5).Infof("Removing gateway VIPs: %v from load balancers: %v", vips, loadBalancers) - txn := util.NewNBTxn() - if err := loadbalancer.DeleteLoadBalancerVIPs(txn, loadBalancers, vips); err != nil { - return err - } - if stdout, stderr, err := txn.Commit(); err != nil { - return fmt.Errorf("error deleting node load balancer %v VIPs %v"+ - "stdout: %q, stderr: %q, error: %v", - loadBalancers, vips, stdout, stderr, err) - } - - return nil + return out, nil } // hasHostEndpoints determines if a slice of endpoints contains a host networked pod func hasHostEndpoints(endpointIPs []string) bool { for _, endpointIP := range endpointIPs { found := false - for _, clusterNet := range config.Default.ClusterSubnets { + for _, clusterNet := range globalconfig.Default.ClusterSubnets { if clusterNet.CIDR.Contains(net.ParseIP(endpointIP)) { found = true break @@ -347,82 +125,11 @@ func hasHostEndpoints(endpointIPs []string) bool { return false } -// getNodeIPs returns the IPs for every node in the cluster for a specific IP family -func getNodeIPs(isIPv6 bool) ([]string, error) { - nodeIPs := []string{} - gatewayRouters, _, err := gateway.GetOvnGateways() - if err != nil { - return nil, err - } - for _, gatewayRouter := range gatewayRouters { - physicalIPs, err := gateway.GetGatewayPhysicalIPs(gatewayRouter) - if err != nil { - klog.Errorf("Gateway router %s does not have physical ip (%v)", gatewayRouter, err) - continue - } - physicalIPs, err = util.MatchAllIPStringFamily(isIPv6, physicalIPs) - if err != nil { - klog.Errorf("Failed to find node ips for gateway: %s that match IP family, IPv6: %t, error: %v", - gatewayRouter, isIPv6, err) - continue - } - nodeIPs = append(nodeIPs, physicalIPs...) - } - return nodeIPs, nil -} - -// collectServiceVIPs collects all the vips associated to a given service -// and returns them as a set. -func collectServiceVIPs(service *v1.Service) sets.String { - isV6Families := getIPFamiliesEnabled() - res := sets.NewString() - for _, ip := range util.GetClusterIPs(service) { - for _, svcPort := range service.Spec.Ports { - vip := util.JoinHostPortInt32(ip, svcPort.Port) - key := virtualIPKey(vip, svcPort.Protocol) - res.Insert(key) - } - } - for _, svcPort := range service.Spec.Ports { - // Node Port - if svcPort.NodePort != 0 { - for _, isIPv6 := range isV6Families { - nodeIPs, err := getNodeIPs(isIPv6) - if err != nil { - klog.Error(err) - continue - } - for _, ip := range nodeIPs { - vip := util.JoinHostPortInt32(ip, svcPort.NodePort) - key := virtualIPKey(vip, svcPort.Protocol) - res.Insert(key) - } - } - } - - for _, extIP := range service.Spec.ExternalIPs { - vip := util.JoinHostPortInt32(extIP, svcPort.Port) - key := virtualIPKey(vip, svcPort.Protocol) - res.Insert(key) - } - // LoadBalancer - for _, ingress := range service.Status.LoadBalancer.Ingress { - if ingress.IP == "" { - continue - } - vip := util.JoinHostPortInt32(ingress.IP, svcPort.Port) - key := virtualIPKey(vip, svcPort.Protocol) - res.Insert(key) - } - } - return res -} - const OvnServiceIdledSuffix = "idled-at" // When idling or empty LB events are enabled, we want to ensure we receive these packets and not reject them. func svcNeedsIdling(annotations map[string]string) bool { - if !config.Kubernetes.OVNEmptyLbEvents { + if !globalconfig.Kubernetes.OVNEmptyLbEvents { return false } @@ -433,16 +140,3 @@ func svcNeedsIdling(annotations map[string]string) bool { } return false } - -// get IPFamiliesEnabled returns a slice representing which ip families -// are enabled, with false representing ipv4, and true for ipv6 -func getIPFamiliesEnabled() []bool { - isV6Families := make([]bool, 0, 2) - if config.IPv4Mode { - isV6Families = append(isV6Families, false) - } - if config.IPv6Mode { - isV6Families = append(isV6Families, true) - } - return isV6Families -} diff --git a/go-controller/pkg/ovn/controller/services/utils_test.go b/go-controller/pkg/ovn/controller/services/utils_test.go index 7a7a57a87f..c0ce2756b6 100644 --- a/go-controller/pkg/ovn/controller/services/utils_test.go +++ b/go-controller/pkg/ovn/controller/services/utils_test.go @@ -4,119 +4,8 @@ import ( "testing" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" - ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/sets" ) -func Test_deleteVIPsFromOVN(t *testing.T) { - config.Kubernetes.OVNEmptyLbEvents = true - defer func() { - config.Kubernetes.OVNEmptyLbEvents = false - }() - - type args struct { - vips sets.String - svc *v1.Service - ovnCmd []ovntest.ExpectedCmd - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "empty", - args: args{ - vips: sets.NewString(), - svc: &v1.Service{}, - ovnCmd: []ovntest.ExpectedCmd{}, - }, - wantErr: false, - }, - { - name: "delete existing vip", - args: args{ - vips: sets.NewString("10.0.0.1:80/TCP"), - svc: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "ns"}, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeClusterIP, - ClusterIP: "10.0.0.1", - ClusterIPs: []string{"10.0.0.1"}, - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{{ - Port: 80, - Protocol: v1.ProtocolTCP, - TargetPort: intstr.FromInt(3456), - }}, - }, - }, - ovnCmd: []ovntest.ExpectedCmd{ - { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_router options:chassis!=null", - Output: gatewayRouter1, - }, - { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: loadbalancerTCP, - }, - { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=2e290f10-3652-11eb-839b-a8a1590cda29", - Output: "loadbalancer1", - }, - { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-worker-lb-tcp=2e290f10-3652-11eb-839b-a8a1590cda29", - Output: "workerlb", - }, - { - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda29 vips "10.0.0.1:80"` + - ` -- --if-exists remove load_balancer loadbalancer1 vips "10.0.0.1:80"` + - ` -- --if-exists remove load_balancer workerlb vips "10.0.0.1:80"`, - Output: "", - }, - { - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-idling-lb-tcp=yes", - Output: idlingloadbalancerTCP, - }, - { - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer a08ea426-2288-11eb-a30b-a8a1590cda30 vips "10.0.0.1:80"`, - Output: "", - }, - }, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - st := newServiceTracker() - if len(tt.args.svc.Spec.ClusterIP) > 0 { - st.updateKubernetesService(tt.args.svc, "") - } - // Expected OVN commands - fexec := ovntest.NewFakeExec() - for _, cmd := range tt.args.ovnCmd { - cmd := cmd - fexec.AddFakeCmd(&cmd) - } - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - if err := deleteVIPsFromAllOVNBalancers(tt.args.vips, tt.args.svc.Name, tt.args.svc.Namespace); (err != nil) != tt.wantErr { - t.Errorf("deleteVIPsFromOVN() error = %v, wantErr %v", err, tt.wantErr) - } - if !fexec.CalledMatchesExpected() { - t.Error(fexec.ErrorDesc()) - } - }) - } -} - func TestServiceNeedsIdling(t *testing.T) { config.Kubernetes.OVNEmptyLbEvents = true defer func() { diff --git a/go-controller/pkg/ovn/gateway.go b/go-controller/pkg/ovn/gateway.go index 11b9c6b8b7..0bc9f63bc7 100644 --- a/go-controller/pkg/ovn/gateway.go +++ b/go-controller/pkg/ovn/gateway.go @@ -4,27 +4,12 @@ import ( "fmt" "net" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/gateway" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - kapi "k8s.io/api/core/v1" "k8s.io/klog/v2" ) -func (ovn *Controller) getGatewayPhysicalIPs(gatewayRouter string) ([]string, error) { - return gateway.GetGatewayPhysicalIPs(gatewayRouter) -} - -func (ovn *Controller) getGatewayLoadBalancer(gatewayRouter string, protocol kapi.Protocol) (string, error) { - return gateway.GetGatewayLoadBalancer(gatewayRouter, protocol) -} - -// getGatewayLoadBalancers find TCP, SCTP, UDP load-balancers from gateway router. -func getGatewayLoadBalancers(gatewayRouter string) (string, string, string, error) { - return gateway.GetGatewayLoadBalancers(gatewayRouter) -} - // getJoinLRPAddresses check if IPs of gateway logical router port are within the join switch IP range, and return them if true. func (oc *Controller) getJoinLRPAddresses(nodeName string) []*net.IPNet { // try to get the IPs from the logical router port diff --git a/go-controller/pkg/ovn/gateway/gateway.go b/go-controller/pkg/ovn/gateway/gateway.go index 5fbd8e22f5..85c90f453d 100644 --- a/go-controller/pkg/ovn/gateway/gateway.go +++ b/go-controller/pkg/ovn/gateway/gateway.go @@ -4,11 +4,8 @@ import ( "fmt" "strings" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "github.com/pkg/errors" - - kapi "k8s.io/api/core/v1" ) const ( @@ -61,41 +58,3 @@ func GetGatewayPhysicalIPs(gatewayRouter string) ([]string, error) { } return []string{physicalIP}, nil } - -// GetGatewayLoadBalancer return the gateway load balancer -func GetGatewayLoadBalancer(gatewayRouter string, protocol kapi.Protocol) (string, error) { - externalIDKey := fmt.Sprintf("%s_%s", string(protocol), OvnGatewayLoadBalancerIds) - loadBalancer, _, err := util.RunOVNNbctl("--data=bare", "--no-heading", - "--columns=_uuid", "find", "load_balancer", - "external_ids:"+externalIDKey+"="+ - gatewayRouter) - if err != nil { - return "", err - } - if loadBalancer == "" { - return "", OVNGatewayLBIsEmpty - } - return loadBalancer, nil -} - -// GetGatewayLoadBalancers find TCP, SCTP, UDP load-balancers from gateway router. -func GetGatewayLoadBalancers(gatewayRouter string) (string, string, string, error) { - lbTCP, stderr, err := util.FindOVNLoadBalancer(types.GatewayLBTCP, gatewayRouter) - if err != nil { - return "", "", "", errors.Wrapf(err, "failed to get gateway router %q TCP "+ - "load balancer, stderr: %q", gatewayRouter, stderr) - } - - lbUDP, stderr, err := util.FindOVNLoadBalancer(types.GatewayLBUDP, gatewayRouter) - if err != nil { - return "", "", "", errors.Wrapf(err, "failed to get gateway router %q UDP "+ - "load balancer, stderr: %q", gatewayRouter, stderr) - } - - lbSCTP, stderr, err := util.FindOVNLoadBalancer(types.GatewayLBSCTP, gatewayRouter) - if err != nil { - return "", "", "", errors.Wrapf(err, "failed to get gateway router %q SCTP "+ - "load balancer, stderr: %q", gatewayRouter, stderr) - } - return lbTCP, lbUDP, lbSCTP, nil -} diff --git a/go-controller/pkg/ovn/gateway_cleanup.go b/go-controller/pkg/ovn/gateway_cleanup.go index e2cb785ee8..c736a8936f 100644 --- a/go-controller/pkg/ovn/gateway_cleanup.go +++ b/go-controller/pkg/ovn/gateway_cleanup.go @@ -61,26 +61,6 @@ func gatewayCleanup(nodeName string) error { "error: %v", exGWexternalSwitch, stderr, err) } - // If exists, remove the TCP, UDP load-balancers created for north-south traffic for gateway router. - k8sNSLbTCP, k8sNSLbUDP, k8sNSLbSCTP, err := getGatewayLoadBalancers(gatewayRouter) - if err != nil { - return err - } - protoLBMap := map[kapi.Protocol]string{ - kapi.ProtocolTCP: k8sNSLbTCP, - kapi.ProtocolUDP: k8sNSLbUDP, - kapi.ProtocolSCTP: k8sNSLbSCTP, - } - for proto, uuid := range protoLBMap { - if uuid != "" { - _, stderr, err = util.RunOVNNbctl("lb-del", uuid) - if err != nil { - return fmt.Errorf("failed to delete Gateway router %s's %s load balancer %s, stderr: %q, "+ - "error: %v", gatewayRouter, proto, uuid, stderr, err) - } - } - } - // We don't know the gateway mode as this is running in the master, try to delete the additional local // gateway for the shared gateway mode. it will be no op if this is done for other gateway modes. delPbrAndNatRules(nodeName, nil) @@ -225,26 +205,6 @@ func multiJoinSwitchGatewayCleanup(nodeName string, upgradeOnly bool) error { "error: %v", externalSwitch, stderr, err) } - // If exists, remove the TCP, UDP load-balancers created for north-south traffic for gateway router. - k8sNSLbTCP, k8sNSLbUDP, k8sNSLbSCTP, err := getGatewayLoadBalancers(gatewayRouter) - if err != nil { - return err - } - protoLBMap := map[kapi.Protocol]string{ - kapi.ProtocolTCP: k8sNSLbTCP, - kapi.ProtocolUDP: k8sNSLbUDP, - kapi.ProtocolSCTP: k8sNSLbSCTP, - } - for proto, uuid := range protoLBMap { - if uuid != "" { - _, stderr, err = util.RunOVNNbctl("lb-del", uuid) - if err != nil { - return fmt.Errorf("failed to delete Gateway router %s's %s load balancer %s, stderr: %q, "+ - "error: %v", gatewayRouter, proto, uuid, stderr, err) - } - } - } - // We don't know the gateway mode as this is running in the master, try to delete the additional local // gateway for the shared gateway mode. it will be no op if this is done for other gateway modes. delPbrAndNatRules(nodeName, nil) diff --git a/go-controller/pkg/ovn/gateway_init.go b/go-controller/pkg/ovn/gateway_init.go index 633b8421ba..145b4a583b 100644 --- a/go-controller/pkg/ovn/gateway_init.go +++ b/go-controller/pkg/ovn/gateway_init.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -127,124 +126,6 @@ func gatewayInit(nodeName string, clusterIPSubnet []*net.IPNet, hostSubnets []*n } } - var k8sNSLbTCP, k8sNSLbUDP, k8sNSLbSCTP string - k8sNSLbTCP, k8sNSLbUDP, k8sNSLbSCTP, err = getGatewayLoadBalancers(gatewayRouter) - if err != nil { - return err - } - gatewayProtoLBMap := map[kapi.Protocol]string{ - kapi.ProtocolTCP: k8sNSLbTCP, - kapi.ProtocolUDP: k8sNSLbUDP, - kapi.ProtocolSCTP: k8sNSLbSCTP, - } - workerK8sNSLbTCP, workerK8sNSLbUDP, workerK8sNSLbSCTP, err := loadbalancer.GetWorkerLoadBalancers(nodeName) - if err != nil { - return err - } - workerProtoLBMap := map[kapi.Protocol]string{ - kapi.ProtocolTCP: workerK8sNSLbTCP, - kapi.ProtocolUDP: workerK8sNSLbUDP, - kapi.ProtocolSCTP: workerK8sNSLbSCTP, - } - enabledProtos := []kapi.Protocol{kapi.ProtocolTCP, kapi.ProtocolUDP} - if sctpSupport { - enabledProtos = append(enabledProtos, kapi.ProtocolSCTP) - } - - if l3GatewayConfig.NodePortEnable { - // Create 3 load-balancers for north-south traffic for each gateway - // router: UDP, TCP, SCTP - for _, proto := range enabledProtos { - if gatewayProtoLBMap[proto] == "" { - gatewayProtoLBMap[proto], stderr, err = util.RunOVNNbctl("--", "create", - "load_balancer", - fmt.Sprintf("external_ids:%s_lb_gateway_router=%s", proto, gatewayRouter), - fmt.Sprintf("protocol=%s", strings.ToLower(string(proto)))) - if err != nil { - return fmt.Errorf("failed to create load balancer for gateway router %s for protocol %s: "+ - "stderr: %q, error: %v", gatewayRouter, proto, stderr, err) - } - } - } - - // Local gateway mode does not use GR for ingress node port traffic, it uses mp0 instead - if config.Gateway.Mode != config.GatewayModeLocal { - // Add north-south load-balancers to the gateway router. - lbString := fmt.Sprintf("%s,%s", gatewayProtoLBMap[kapi.ProtocolTCP], gatewayProtoLBMap[kapi.ProtocolUDP]) - if sctpSupport { - lbString = lbString + "," + gatewayProtoLBMap[kapi.ProtocolSCTP] - } - stdout, stderr, err = util.RunOVNNbctl("set", "logical_router", gatewayRouter, "load_balancer="+lbString) - if err != nil { - return fmt.Errorf("failed to set north-south load-balancers to the "+ - "gateway router %s, stdout: %q, stderr: %q, error: %v", - gatewayRouter, stdout, stderr, err) - } - } else { - // Also add north-south load-balancers to local switches for pod -> nodePort traffic - stdout, stderr, err = util.RunOVNNbctl("get", "logical_switch", nodeName, "load_balancer") - if err != nil { - return fmt.Errorf("failed to get load-balancers on the node switch %s, stdout: %q, "+ - "stderr: %q, error: %v", nodeName, stdout, stderr, err) - } - for _, proto := range enabledProtos { - if !strings.Contains(stdout, gatewayProtoLBMap[proto]) { - stdout, stderr, err = util.RunOVNNbctl("ls-lb-add", nodeName, gatewayProtoLBMap[proto]) - if err != nil { - return fmt.Errorf("failed to add north-south load-balancer %s to the "+ - "node switch %s, stdout: %q, stderr: %q, error: %v", - gatewayProtoLBMap[proto], nodeName, stdout, stderr, err) - } - } - } - } - } - - // Create load balancers for workers (to be applied to GR and node switch) - for _, proto := range enabledProtos { - if workerProtoLBMap[proto] == "" { - workerProtoLBMap[proto], stderr, err = util.RunOVNNbctl("--", "create", - "load_balancer", - fmt.Sprintf("external_ids:%s-%s=%s", types.WorkerLBPrefix, strings.ToLower(string(proto)), nodeName), - fmt.Sprintf("protocol=%s", strings.ToLower(string(proto)))) - if err != nil { - return fmt.Errorf("failed to create load balancer for worker node %s for protocol %s: "+ - "stderr: %q, error: %v", nodeName, proto, stderr, err) - } - } - } - - if config.Gateway.Mode != config.GatewayModeLocal { - // Ensure north-south load-balancers are not on local switches for pod -> nodePort traffic - // For upgrade path REMOVEME later - stdout, stderr, err = util.RunOVNNbctl("get", "logical_switch", nodeName, "load_balancer") - if err != nil { - return fmt.Errorf("failed to get load-balancers on the node switch %s, stdout: %q, "+ - "stderr: %q, error: %v", nodeName, stdout, stderr, err) - } - for _, proto := range enabledProtos { - if strings.Contains(stdout, gatewayProtoLBMap[proto]) { - _, stderr, err = util.RunOVNNbctl("ls-lb-del", nodeName, gatewayProtoLBMap[proto]) - if err != nil { - return fmt.Errorf("failed to remove north-south load-balancer %s to the "+ - "node switch %s, stderr: %q, error: %v", - gatewayProtoLBMap[proto], nodeName, stderr, err) - } - } - } - - // Add per worker switch specific load-balancers - for _, proto := range enabledProtos { - if !strings.Contains(stdout, workerProtoLBMap[proto]) { - _, stderr, err = util.RunOVNNbctl("ls-lb-add", nodeName, workerProtoLBMap[proto]) - if err != nil { - return fmt.Errorf("failed to add worker load-balancer %s to the "+ - "node switch %s, stderr: %q, error: %v", - workerProtoLBMap[proto], nodeName, stderr, err) - } - } - } - } if err := addExternalSwitch("", l3GatewayConfig.InterfaceID, nodeName, diff --git a/go-controller/pkg/ovn/gateway_test.go b/go-controller/pkg/ovn/gateway_test.go index 6c1b42ffda..8095d3f848 100644 --- a/go-controller/pkg/ovn/gateway_test.go +++ b/go-controller/pkg/ovn/gateway_test.go @@ -80,43 +80,6 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, "ovn-nbctl --timeout=15 --may-exist lr-route-add GR_test-node 10.128.0.0/14 100.64.0.1", }) - const ( - tcpLBUUID string = "1a3dfc82-2749-4931-9190-c30e7c0ecea3" - udpLBUUID string = "6d3142fc-53e8-4ac1-88e6-46094a5a9957" - ) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBTCP + "=GR_test-node", - Output: tcpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBUDP + "=GR_test-node", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBSCTP + "=GR_test-node", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBTCP + "=test-node", - Output: tcpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBUDP + "=test-node", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBSCTP + "=test-node", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.GatewayLBUDP + "=GR_test-node protocol=udp", - Output: udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 set logical_router GR_test-node load_balancer=" + tcpLBUUID + "," + udpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.WorkerLBUDP + "=test-node protocol=udp", - Output: udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 get logical_switch test-node load_balancer", - "ovn-nbctl --timeout=15 ls-lb-add test-node " + tcpLBUUID, - "ovn-nbctl --timeout=15 ls-lb-add test-node " + udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 --may-exist ls-add ext_test-node", "ovn-nbctl --timeout=15 -- --may-exist lsp-add ext_test-node INTERFACE-ID -- lsp-set-addresses INTERFACE-ID unknown -- lsp-set-type INTERFACE-ID localnet -- lsp-set-options INTERFACE-ID network_name=physnet", @@ -169,43 +132,6 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, "ovn-nbctl --timeout=15 --may-exist lr-route-add GR_test-node fd01::/48 fd98::1", }) - const ( - tcpLBUUID string = "1a3dfc82-2749-4931-9190-c30e7c0ecea3" - udpLBUUID string = "6d3142fc-53e8-4ac1-88e6-46094a5a9957" - ) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBTCP + "=GR_test-node", - Output: tcpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBUDP + "=GR_test-node", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBSCTP + "=GR_test-node", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBTCP + "=test-node", - Output: tcpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBUDP + "=test-node", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBSCTP + "=test-node", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.GatewayLBUDP + "=GR_test-node protocol=udp", - Output: udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 set logical_router GR_test-node load_balancer=" + tcpLBUUID + "," + udpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.WorkerLBUDP + "=test-node protocol=udp", - Output: udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 get logical_switch test-node load_balancer", - "ovn-nbctl --timeout=15 ls-lb-add test-node " + tcpLBUUID, - "ovn-nbctl --timeout=15 ls-lb-add test-node " + udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 --may-exist ls-add ext_test-node", "ovn-nbctl --timeout=15 -- --may-exist lsp-add ext_test-node INTERFACE-ID -- lsp-set-addresses INTERFACE-ID unknown -- lsp-set-type INTERFACE-ID localnet -- lsp-set-options INTERFACE-ID network_name=physnet", @@ -257,43 +183,6 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, "ovn-nbctl --timeout=15 --may-exist lr-route-add GR_test-node fd01::/48 fd98::1", }) - const ( - tcpLBUUID string = "1a3dfc82-2749-4931-9190-c30e7c0ecea3" - udpLBUUID string = "6d3142fc-53e8-4ac1-88e6-46094a5a9957" - ) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBTCP + "=GR_test-node", - Output: tcpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBUDP + "=GR_test-node", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBSCTP + "=GR_test-node", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBTCP + "=test-node", - Output: tcpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBUDP + "=test-node", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBSCTP + "=test-node", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.GatewayLBUDP + "=GR_test-node protocol=udp", - Output: udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 set logical_router GR_test-node load_balancer=" + tcpLBUUID + "," + udpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.WorkerLBUDP + "=test-node protocol=udp", - Output: udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 get logical_switch test-node load_balancer", - "ovn-nbctl --timeout=15 ls-lb-add test-node " + tcpLBUUID, - "ovn-nbctl --timeout=15 ls-lb-add test-node " + udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 --may-exist ls-add ext_test-node", "ovn-nbctl --timeout=15 -- --may-exist lsp-add ext_test-node INTERFACE-ID -- lsp-set-addresses INTERFACE-ID unknown -- lsp-set-type INTERFACE-ID localnet -- lsp-set-options INTERFACE-ID network_name=physnet", @@ -323,7 +212,6 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, hostSubnet := ovntest.MustParseIPNet("10.130.0.0/23") const ( nodeRouteUUID string = "0cac12cf-3e0f-4682-b028-5ea2e0001962" - tcpLBUUID string = "1a3dfc82-2749-4931-9190-c30e7c0ecea3" ) fexec := ovntest.NewFakeExec() @@ -348,21 +236,6 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, "ovn-nbctl --timeout=15 --if-exist ls-del ext_ext_test-node", }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_test-node", - Output: tcpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:UDP_lb_gateway_router=GR_test-node", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:SCTP_lb_gateway_router=GR_test-node", - Output: "", - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 lb-del " + tcpLBUUID, - }) cleanupPBRandNATRules(fexec, nodeName, []*net.IPNet{hostSubnet}) err = gatewayCleanup(nodeName) @@ -378,7 +251,6 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, v6RouteUUID string = "0cac12cf-4682-3e0f-b028-5ea2e0001962" v4mgtRouteUUID string = "0cac12cf-3e0f-4682-b028-5ea2e0001963" v6mgtRouteUUID string = "0cac12cf-4682-3e0f-b028-5ea2e0001963" - tcpLBUUID string = "1a3dfc82-2749-4931-9190-c30e7c0ecea3" ) fexec := ovntest.NewFakeExec() @@ -411,21 +283,6 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, "ovn-nbctl --timeout=15 --if-exist ls-del ext_ext_test-node", }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=GR_test-node", - Output: tcpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:UDP_lb_gateway_router=GR_test-node", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:SCTP_lb_gateway_router=GR_test-node", - Output: "", - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 lb-del " + tcpLBUUID, - }) cleanupPBRandNATRules(fexec, nodeName, hostSubnets) err = gatewayCleanup(nodeName) @@ -466,43 +323,6 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, "ovn-nbctl --timeout=15 --may-exist lr-route-add GR_test-node 10.128.0.0/14 100.64.0.1", }) - const ( - tcpLBUUID string = "1a3dfc82-2749-4931-9190-c30e7c0ecea3" - udpLBUUID string = "6d3142fc-53e8-4ac1-88e6-46094a5a9957" - ) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBTCP + "=GR_test-node", - Output: tcpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBUDP + "=GR_test-node", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBSCTP + "=GR_test-node", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBTCP + "=test-node", - Output: tcpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBUDP + "=test-node", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBSCTP + "=test-node", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.GatewayLBUDP + "=GR_test-node protocol=udp", - Output: udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 set logical_router GR_test-node load_balancer=" + tcpLBUUID + "," + udpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.WorkerLBUDP + "=test-node protocol=udp", - Output: udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 get logical_switch test-node load_balancer", - "ovn-nbctl --timeout=15 ls-lb-add test-node " + tcpLBUUID, - "ovn-nbctl --timeout=15 ls-lb-add test-node " + udpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 --may-exist ls-add ext_test-node", "ovn-nbctl --timeout=15 -- --may-exist lsp-add ext_test-node INTERFACE-ID -- lsp-set-addresses INTERFACE-ID unknown -- lsp-set-type INTERFACE-ID localnet -- lsp-set-options INTERFACE-ID network_name=physnet", diff --git a/go-controller/pkg/ovn/loadbalancer/lb_cache.go b/go-controller/pkg/ovn/loadbalancer/lb_cache.go new file mode 100644 index 0000000000..3ea1705b60 --- /dev/null +++ b/go-controller/pkg/ovn/loadbalancer/lb_cache.go @@ -0,0 +1,354 @@ +package loadbalancer + +import ( + "encoding/json" + "fmt" + "strings" + "sync" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "k8s.io/apimachinery/pkg/util/sets" +) + +var globalCache *LBCache +var globalCacheLock sync.Mutex = sync.Mutex{} + +// GetLBCache returns the global load balancer cache, and initializes it +// if not yet set. +func GetLBCache() (*LBCache, error) { + globalCacheLock.Lock() + defer globalCacheLock.Unlock() + + if globalCache != nil { + return globalCache, nil + } + + c, err := newCache() + if err != nil { + return nil, err + } + globalCache = c + return globalCache, nil +} + +// LBCache caches the state of load balancers in ovn. +// It is used to prevent unnecessary accesses to the database +type LBCache struct { + sync.Mutex + + existing map[string]*CachedLB +} + +// we don't need to store / populate all information, just a subset +type CachedLB struct { + Name string + Protocol string + UUID string + ExternalIDs map[string]string + VIPs sets.String // don't care about backend IPs, just the vips + + Switches sets.String + Routers sets.String +} + +// update the database with any existing LBs, along with any +// that were deleted. +func (c *LBCache) update(existing []LB, toDelete []string) { + c.Lock() + defer c.Unlock() + for _, uuid := range toDelete { + delete(c.existing, uuid) + } + + for _, lb := range existing { + if lb.UUID == "" { + panic("coding error: cache add LB with no UUID") + } + c.existing[lb.UUID] = &CachedLB{ + Name: lb.Name, + UUID: lb.UUID, + Protocol: strings.ToLower(lb.Protocol), + ExternalIDs: lb.ExternalIDs, + VIPs: getVips(&lb), + + Switches: sets.NewString(lb.Switches...), + Routers: sets.NewString(lb.Routers...), + } + } +} + +// removeVIPs updates the cache after a successful DeleteLoadBalancerVIPs call +func (c *LBCache) removeVips(toRemove []DeleteVIPEntry) { + c.Lock() + defer c.Unlock() + + for _, entry := range toRemove { + lb := c.existing[entry.LBUUID] + if lb == nil { + continue + } + + // lb is a pointer, this is immediately effecting. + lb.VIPs.Delete(entry.VIPs...) + } +} + +// addNewLB is a shortcut when creating a load balancer; we know it won't have any switches or routers +func (c *LBCache) addNewLB(lb *LB) { + c.Lock() + defer c.Unlock() + if lb.UUID == "" { + panic("coding error! LB has empty UUID") + } + c.existing[lb.UUID] = &CachedLB{ + Name: lb.Name, + UUID: lb.UUID, + Protocol: strings.ToLower(lb.Protocol), + ExternalIDs: lb.ExternalIDs, + VIPs: getVips(lb), + + Switches: sets.NewString(), + Routers: sets.NewString(), + } +} + +func getVips(lb *LB) sets.String { + out := sets.NewString() + for _, rule := range lb.Rules { + out.Insert(rule.Source.String()) + } + return out +} + +// Find searches through the cache for load balancers that match the list of external IDs. +// It returns all found load balancers, indexed by uuid. +func (c *LBCache) Find(externalIDs map[string]string) map[string]*CachedLB { + c.Lock() + defer c.Unlock() + + out := map[string]*CachedLB{} + + for uuid, lb := range c.existing { + if extIDsMatch(externalIDs, lb.ExternalIDs) { + out[uuid] = lb + } + } + + return out +} + +// extIDsMatch returns true if have is a superset of want. +func extIDsMatch(want, have map[string]string) bool { + for k, v := range want { + actual, ok := have[k] + if !ok { + return false + } + if actual != v { + return false + } + } + + return true +} + +// newCache creates a lbCache, populated with all existing load balancers +func newCache() (*LBCache, error) { + // first, list all load balancers + lbs, err := listLBs() + if err != nil { + return nil, err + } + + c := LBCache{} + c.existing = make(map[string]*CachedLB, len(lbs)) + + for i := range lbs { + c.existing[lbs[i].UUID] = &lbs[i] + } + + switches, err := findTableLBs("logical_switch") + if err != nil { + return nil, err + } + + for switchname, lbuuids := range switches { + for _, lbuuid := range lbuuids { + if lb, ok := c.existing[lbuuid]; ok { + lb.Switches.Insert(switchname) + } + } + } + + routers, err := findTableLBs("logical_router") + if err != nil { + return nil, err + } + + for routername, lbuuids := range routers { + for _, lbuuid := range lbuuids { + if lb, ok := c.existing[lbuuid]; ok { + lb.Routers.Insert(routername) + } + } + } + + return &c, nil +} + +// listLBs lists all load balancers in nbdb +func listLBs() ([]CachedLB, error) { + stdout, _, err := util.RunOVNNbctlRawOutput(15, "--format=json", "--data=json", + "--columns=name,_uuid,protocol,external_ids,vips", "find", "load_balancer") + + if err != nil { + return nil, fmt.Errorf("could not list load_balancer: %w", err) + } + + data := struct { + Data [][]interface{} `json:"data"` + Headings []string `json:"headings"` + }{} + + err = json.Unmarshal([]byte(stdout), &data) + if err != nil { + return nil, fmt.Errorf("failed to parse find load_balancer response: %w", err) + } + + out := make([]CachedLB, 0, len(data.Data)) + + for _, row := range data.Data { + if len(row) != 5 { + return nil, fmt.Errorf("failed to parse find load_balancer response: wrong number of columns (want 5) in row %#v", row) + } + + res := CachedLB{ + VIPs: sets.String{}, + Switches: sets.String{}, + Routers: sets.String{}, + } + // parse the row + + // name + if str, ok := row[0].(string); ok { + res.Name = str + } else { + return nil, fmt.Errorf("failed to parse find load_balancer response: name: expected string, got %#v", row[0]) + } + + // uuid is a pair + if cell, ok := row[1].([]interface{}); ok { + if len(cell) != 2 { + return nil, fmt.Errorf("failed to parse find load_balancer response: uuid: expected pair, got %#v", cell) + } else { + if str, ok := cell[1].(string); ok { + res.UUID = str + } else { + return nil, fmt.Errorf("failed to parse find load_balancer response: uuid: expected string, got %#v", cell[1]) + } + } + } else { + return nil, fmt.Errorf("failed to parse find load_balancer response: uuid: expected slice, got %#v", row[1]) + } + + // protocol may be a string or an empty set + if str, ok := row[2].(string); ok { + res.Protocol = str + } else if _, ok := row[2].([]interface{}); ok { //empty set + // empty protocol, assume tcp + res.Protocol = "tcp" + } else { + return nil, fmt.Errorf("failed to parse find load_balancer response: protocol: expected string, got %#v", row[2]) + } + + res.ExternalIDs, err = extractMap(row[3]) + if err != nil { + return nil, fmt.Errorf("failed to parse find load_balancer response: external_ids: %w", err) + } + + vips, err := extractMap(row[4]) + if err != nil { + return nil, fmt.Errorf("failed to parse find load_balancer response: vips: %w", err) + } + for vip := range vips { + res.VIPs.Insert(vip) + } + out = append(out, res) + } + + return out, nil +} + +// findTableLBs returns a list of router name -> lb uuids +func findTableLBs(tablename string) (map[string][]string, error) { + // this is CSV and not JSON because ovn-nbctl is inconsistent when a set has a single value :-/ + rows, err := util.RunOVNNbctlCSV([]string{"--data=bare", "--columns=name,load_balancer", "find", tablename}) + if err != nil { + return nil, fmt.Errorf("failed to find existing LBs for %s: %w", tablename, err) + } + + out := make(map[string][]string, len(rows)) + for _, row := range rows { + if len(row) != 2 { + return nil, fmt.Errorf("invalid row returned when listing LBs for %s: %#v", tablename, row) + } + if row[0] == "" { + continue + } + + lbs := strings.Split(row[1], " ") + if row[1] == "" { + lbs = []string{} + } + out[row[0]] = lbs + } + + return out, nil +} + +func TestOnlySetCache(cache *LBCache) { + globalCache = cache +} + +// extractMap converts an untyped ovn json map in to a real map +// it looks like this: +// [ "map", [ ["k1", "v1"], ["k2", "v2"] ]] +func extractMap(in interface{}) (map[string]string, error) { + out := map[string]string{} + + // ["map", [ pairs]] + if cell, ok := in.([]interface{}); ok { + if len(cell) != 2 { + return nil, fmt.Errorf("expected outer pair, got %#v", cell) + } else { + // rows: [ [k,v], [k, v], ...] + if rows, ok := cell[1].([]interface{}); ok { + for _, row := range rows { + if pair, ok := row.([]interface{}); ok { + if len(pair) != 2 { + return nil, fmt.Errorf("expected k-v pair, got %#v", pair) + } else { + k, ok := pair[0].(string) + if !ok { + return nil, fmt.Errorf("key not a string: %#v", pair[0]) + } + v, ok := pair[1].(string) + if !ok { + return nil, fmt.Errorf("value not a string: %#v", pair[1]) + } + + out[k] = v + } + } else { + return nil, fmt.Errorf("expected k-v slice, got %#v", row) + } + } + } else { + return nil, fmt.Errorf("expected map list, got %#v", cell[1]) + } + } + } else { + return nil, fmt.Errorf("expected outer slice, got %#v", in) + } + return out, nil +} diff --git a/go-controller/pkg/ovn/loadbalancer/lb_cache_test.go b/go-controller/pkg/ovn/loadbalancer/lb_cache_test.go new file mode 100644 index 0000000000..ae1770eb0e --- /dev/null +++ b/go-controller/pkg/ovn/loadbalancer/lb_cache_test.go @@ -0,0 +1,160 @@ +package loadbalancer + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/sets" + + ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +func TestNewCache(t *testing.T) { + + data := ` +{ + "data": [ + [ + "Service_default/kubernetes_TCP_node_router_ovn-control-plane", + [ + "uuid", + "cb6ebcb0-c12d-4404-ada7-5aa2b898f06b" + ], + "tcp", + [ + "map", + [ + [ + "k8s.ovn.org/kind", + "Service" + ], + [ + "k8s.ovn.org/owner", + "default/kubernetes" + ] + ] + ], + [ + "map", + [ + [ + "192.168.0.1:6443", + "1.1.1.1:1,2.2.2.2:2" + ], + [ + "[fe::1]:1", + "[fe::2]:1,[fe::2]:2" + ] + ] + ] + ], + [ + "Service_default/kubernetes_TCP_node_switch_ovn-control-plane_merged", + [ + "uuid", + "7dc190c4-c615-467f-af83-9856d832c9a0" + ], + "tcp", + [ + "map", + [ + [ + "k8s.ovn.org/kind", + "Service" + ], + [ + "k8s.ovn.org/owner", + "default/kubernetes" + ] + ] + ], + [ + "map", + [ + [ + "192.168.0.1:6443", + "1.1.1.1:1,2.2.2.2:2" + ], + [ + "[ff::1]:1", + "[fe::2]:1,[fe::2]:2" + ] + ] + ] + ] + ], + "headings": [ + "name", + "_uuid", + "external_ids", + "protocol" + ] +} +` + + fexec := ovntest.NewFakeExec() + fexec.AddFakeCmd(&ovntest.ExpectedCmd{ + Cmd: "ovn-nbctl --timeout=15 --format=json --data=json --columns=name,_uuid,protocol,external_ids,vips find load_balancer", + Output: data, + }) + + fexec.AddFakeCmd(&ovntest.ExpectedCmd{ + Cmd: `ovn-nbctl --timeout=15 --no-heading --format=csv --data=bare --columns=name,load_balancer find logical_router`, + Output: `GR_ovn-worker2,31bb6bff-93b9-4080-a1b9-9a1fa898b1f0 7dc190c4-c615-467f-af83-9856d832c9a0 f0747ebb-71c2-4249-bdca-f33670ae544f +GR_ovn-worker,31bb6bff-93b9-4080-a1b9-9a1fa898b1f0 7dc190c4-c615-467f-af83-9856d832c9a0 f0747ebb-71c2-4249-bdca-f33670ae544f +ovn_cluster_router, +GR_ovn-control-plane,31bb6bff-93b9-4080-a1b9-9a1fa898b1f0 cb6ebcb0-c12d-4404-ada7-5aa2b898f06b f0747ebb-71c2-4249-bdca-f33670ae544f +`, + }) + err := util.SetExec(fexec) + if err != nil { + t.Fatal(err) + } + + lbs, err := listLBs() + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, []CachedLB{ + { + UUID: "cb6ebcb0-c12d-4404-ada7-5aa2b898f06b", + Name: "Service_default/kubernetes_TCP_node_router_ovn-control-plane", + Protocol: "tcp", + ExternalIDs: map[string]string{ + "k8s.ovn.org/kind": "Service", + "k8s.ovn.org/owner": "default/kubernetes", + }, + VIPs: sets.NewString("192.168.0.1:6443", "[fe::1]:1"), + + Switches: sets.String{}, + Routers: sets.String{}, + }, + { + UUID: "7dc190c4-c615-467f-af83-9856d832c9a0", + Name: "Service_default/kubernetes_TCP_node_switch_ovn-control-plane_merged", + Protocol: "tcp", + ExternalIDs: map[string]string{ + "k8s.ovn.org/kind": "Service", + "k8s.ovn.org/owner": "default/kubernetes", + }, + VIPs: sets.NewString("192.168.0.1:6443", "[ff::1]:1"), + + Switches: sets.String{}, + Routers: sets.String{}, + }, + }, lbs) + + routerLBs, err := findTableLBs("logical_router") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, routerLBs, map[string][]string{ + "GR_ovn-worker2": {"31bb6bff-93b9-4080-a1b9-9a1fa898b1f0", "7dc190c4-c615-467f-af83-9856d832c9a0", "f0747ebb-71c2-4249-bdca-f33670ae544f"}, + "GR_ovn-worker": {"31bb6bff-93b9-4080-a1b9-9a1fa898b1f0", "7dc190c4-c615-467f-af83-9856d832c9a0", "f0747ebb-71c2-4249-bdca-f33670ae544f"}, + "ovn_cluster_router": {}, + "GR_ovn-control-plane": {"31bb6bff-93b9-4080-a1b9-9a1fa898b1f0", "cb6ebcb0-c12d-4404-ada7-5aa2b898f06b", "f0747ebb-71c2-4249-bdca-f33670ae544f"}, + }) + +} diff --git a/go-controller/pkg/ovn/loadbalancer/loadbalancer.go b/go-controller/pkg/ovn/loadbalancer/loadbalancer.go index 6be7710200..e1d24b0358 100644 --- a/go-controller/pkg/ovn/loadbalancer/loadbalancer.go +++ b/go-controller/pkg/ovn/loadbalancer/loadbalancer.go @@ -1,332 +1,303 @@ package loadbalancer import ( - "encoding/json" "fmt" + "sort" "strings" - "github.com/pkg/errors" - utilnet "k8s.io/utils/net" + "k8s.io/apimachinery/pkg/util/sets" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - kapi "k8s.io/api/core/v1" "k8s.io/klog/v2" ) -const ( - // used to indicate if the service is running on node load balancers - NodeLoadBalancer = "NodeLoadBalancer" - // used to indicate if the service is running on idling load balancers - IdlingLoadBalancer = "IdlingLoadBalancer" -) +// EnsureLBs provides a generic load-balancer reconciliation engine. +// +// It assures that, for a given set of ExternalIDs, only the configured +// list of load balancers exist. Existing load-balancers will be updated, +// new ones will be created as needed, and stale ones will be deleted. +// +// For example, you might want to ensure that service ns/foo has the +// correct set of load balancers. You would call it with something like +// +// EnsureLBs( { kind: Service, owner: ns/foo}, { {Name: Service_ns/foo_cluster_tcp, ...}}) +// +// This will ensure that, for this example, only that one LB exists and +// has the desired configuration. +// +// It will commit all updates in a single transaction, so updates will be +// atomic and users should see no disruption. However, concurrent calls +// that modify the same externalIDs are not allowed. +// +// It is assumed that names are meaningful and somewhat stable, to minimize churn. This +// function doesn't work with Load_Balancers without a name. +func EnsureLBs(externalIDs map[string]string, LBs []LB) error { + lbCache, err := GetLBCache() + if err != nil { + return fmt.Errorf("failed initialize LBcache: %w", err) + } -type NotFoundError struct { - What string -} + existing := lbCache.Find(externalIDs) + existingByName := make(map[string]*CachedLB, len(existing)) + toDelete := sets.NewString() -func (e NotFoundError) Error() string { - return e.What + " not found" -} + for _, lb := range existing { + existingByName[lb.Name] = lb + toDelete.Insert(lb.UUID) + } -var LBNotFound = NotFoundError{What: "Load balancer"} + createdUUIDs := sets.NewString() -// GetOVNKubeLoadBalancer returns the LoadBalancer matching the protocol -func GetOVNKubeLoadBalancer(protocol kapi.Protocol) (string, error) { - return getLoadBalancerByProtocolType(protocol, types.ClusterLBPrefix) -} + txn := util.NewNBTxn() -// GetOVNKubeIdlingLoadBalancer returns the LoadBalancer matching the protocol -func GetOVNKubeIdlingLoadBalancer(protocol kapi.Protocol) (string, error) { - return getLoadBalancerByProtocolType(protocol, types.ClusterIdlingLBPrefix) -} + // create or update each LB, logging UUID created (so we can clean up) + // missing load balancers will be created immediately + // existing load-balancers will be updated in the transaction + for i, lb := range LBs { + uuid, err := ensureLB(txn, lbCache, &lb, existingByName[lb.Name]) + if err != nil { + return err + } + createdUUIDs.Insert(uuid) + toDelete.Delete(uuid) + LBs[i].UUID = uuid + } -func getLoadBalancerByProtocolType(protocol kapi.Protocol, idkey string) (string, error) { - id, value := fmt.Sprintf("%s-%s", idkey, strings.ToLower(string(protocol))), "yes" - res, _, err := util.FindOVNLoadBalancer(id, value) - if err != nil { - return "", errors.Wrapf(err, "Failed to get ovnkube balancer %v %s", protocol, idkey) + uuidsToDelete := make([]string, 0, len(toDelete)) + for uuid := range toDelete { + uuidsToDelete = append(uuidsToDelete, uuid) } - if res == "" { - return "", LBNotFound + if err := DeleteLBs(txn, uuidsToDelete); err != nil { + return fmt.Errorf("failed to delete %d stale load balancers for %#v: %w", + len(uuidsToDelete), externalIDs, err) } - return res, nil -} -// CreateLoadBalancer creates the loadbalancer if it doesn´t exist to avoid -// consumers to create duplicate loadbalancers with the same name and externalID -func CreateLoadBalancer(protocol kapi.Protocol, idkey string) (string, error) { - lbUUID, err := getLoadBalancerByProtocolType(protocol, idkey) - if err != nil && !errors.Is(err, LBNotFound) { - return "", errors.Wrapf(err, "Failed to get OVN load balancer for protocol %s", protocol) + _, _, err = txn.Commit() + if err != nil { + return fmt.Errorf("failed to commit load balancer changes for %#v: %w", externalIDs, err) } - // create the load balancer if it doesn't exist yet - if lbUUID == "" { - lbUUID, err = createLoadBalancer(protocol, idkey) + klog.V(5).Infof("Deleted %d stale LBs for %#v", len(uuidsToDelete), externalIDs) + + lbCache.update(LBs, uuidsToDelete) + return nil +} + +// ensureLB creates or updates a load balancer as necessary. +// TODO: make this use libovsdb and generally be more efficient +// returns the uuid of the LB +func ensureLB(txn *util.NBTxn, lbCache *LBCache, lb *LB, existing *CachedLB) (string, error) { + uuid := "" + if existing == nil { + cmds := []string{ + "create", "load_balancer", + } + cmds = append(cmds, lbToColumns(lb)...) + // note: load-balancer creation is not in the transaction + stdout, _, err := util.RunOVNNbctl(cmds...) if err != nil { - return "", errors.Wrapf(err, "Failed to create OVN load balancer for protocol %s", protocol) + return "", fmt.Errorf("failed to create load_balancer %s: %w", lb.Name, err) } + uuid = stdout + lb.UUID = uuid + + // Since this short-cut the transation, immediately add it to the cache. + lbCache.addNewLB(lb) + } else { + cmds := []string{ + "set", "load_balancer", existing.UUID, + } + cmds = append(cmds, lbToColumns(lb)...) + _, _, err := txn.AddOrCommit(cmds) + if err != nil { + return "", fmt.Errorf("failed to update load_balancer %s: %w", lb.Name, err) + } + uuid = existing.UUID + lb.UUID = uuid } - return lbUUID, nil -} -// createLoadBalancer creates a loadbalancer for the specified protocol -// all loadbalancers but idling ones reject packets for vips without endpoints by default -func createLoadBalancer(protocol kapi.Protocol, idkey string) (string, error) { - id := fmt.Sprintf("external_ids:%s-%s=yes", idkey, strings.ToLower(string(protocol))) - proto := fmt.Sprintf("protocol=%s", strings.ToLower(string(protocol))) - reject := true - if idkey == types.ClusterIdlingLBPrefix { - reject = false - } - options := fmt.Sprintf("options:reject=%t", reject) - lbID, stderr, err := util.RunOVNNbctl("create", "load_balancer", id, proto, options) - if err != nil { - klog.Errorf("Failed to create %s load balancer, stderr: %q, error: %v", protocol, stderr, err) - return "", err + // List existing routers and switches, to see if there are any for which we should remove + existingRouters := sets.String{} + existingSwitches := sets.String{} + if existing != nil { + existingRouters = existing.Routers + existingSwitches = existing.Switches } - return lbID, nil -} -// GetLoadBalancerVIPs returns a map whose keys are VIPs (IP:port) on loadBalancer -func GetLoadBalancerVIPs(loadBalancer string) (map[string]string, error) { - var vips map[string]string - outStr, _, err := util.RunOVNNbctl("--data=bare", "--no-heading", - "get", "load_balancer", loadBalancer, "vips") - if err != nil { - return nil, err - } - if outStr == "" { - return vips, nil + wantRouters := sets.NewString(lb.Routers...) + wantSwitches := sets.NewString(lb.Switches...) + + // add missing switches + for _, sw := range wantSwitches.Difference(existingSwitches).List() { + _, _, err := txn.AddOrCommit([]string{"--may-exist", "ls-lb-add", sw, uuid}) + if err != nil { + return uuid, fmt.Errorf("failed to synchronize LB %s switches / routers: %w", lb.Name, err) + } } - // sample outStr: - // - {"192.168.0.1:80"="10.1.1.1:80,10.2.2.2:80"} - // - {"[fd01::]:80"="[fd02::]:80,[fd03::]:80"} - outStrMap := strings.Replace(outStr, "=", ":", -1) - err = json.Unmarshal([]byte(outStrMap), &vips) - if err != nil { - return nil, err + // remove old switches + for _, sw := range existingSwitches.Difference(wantSwitches).List() { + _, _, err := txn.AddOrCommit([]string{"--if-exists", "ls-lb-del", sw, uuid}) + if err != nil { + return uuid, fmt.Errorf("failed to synchronize LB %s switches / routers: %w", lb.Name, err) + } } - return vips, nil -} -// DeleteLoadBalancerVIP removes the VIP as well as any reject ACLs associated to the LB -func DeleteLoadBalancerVIP(txn *util.NBTxn, loadBalancer, vip string) error { - vipQuotes := fmt.Sprintf("\"%s\"", vip) - request := []string{"--if-exists", "remove", "load_balancer", loadBalancer, "vips", vipQuotes} - stdout, stderr, err := txn.AddOrCommit(request) - if err != nil { - // if we hit an error and fail to remove load balancer, we skip removing the rejectACL - return fmt.Errorf("error in deleting load balancer vip %s for %s"+ - "stdout: %q, stderr: %q, error: %v", - vip, loadBalancer, stdout, stderr, err) + // add missing routers + for _, rtr := range wantRouters.Difference(existingRouters).List() { + _, _, err := txn.AddOrCommit([]string{"--may-exist", "lr-lb-add", rtr, uuid}) + if err != nil { + return uuid, fmt.Errorf("failed to synchronize LB %s switches / routers: %w", lb.Name, err) + } } - return nil -} + // remove old routers + for _, rtr := range existingRouters.Difference(wantRouters).List() { + _, _, err := txn.AddOrCommit([]string{"--if-exists", "lr-lb-del", rtr, uuid}) + if err != nil { -// DeleteLoadBalancerVIPs removes the VIPs across lbs in a single shot -func DeleteLoadBalancerVIPs(txn *util.NBTxn, loadBalancers, vips []string) error { - for _, loadBalancer := range loadBalancers { - for _, vip := range vips { - if err := DeleteLoadBalancerVIP(txn, loadBalancer, vip); err != nil { - return err - } + return uuid, fmt.Errorf("failed to synchronize LB %s switches / routers: %w", lb.Name, err) } } - return nil + + return uuid, nil } -// UpdateLoadBalancer updates the VIP for sourceIP:sourcePort to point to targets (an -// array of IP:port strings) -func UpdateLoadBalancer(lb, vip string, targets []string) error { - lbTarget := fmt.Sprintf(`vips:"%s"="%s"`, vip, strings.Join(targets, ",")) +// lbToColumns turns a load balancer in to a set of column arguments +// that can be passed to nbctl create or set +func lbToColumns(lb *LB) []string { + reject := "true" + event := "false" - out, stderr, err := util.RunOVNNbctl("set", "load_balancer", lb, lbTarget) - if err != nil { - return fmt.Errorf("error in configuring load balancer: %s "+ - "stdout: %q, stderr: %q, error: %v", lb, out, stderr, err) + if lb.Opts.Unidling { + reject = "false" + event = "true" } - return nil -} - -// GetLogicalSwitchesForLoadBalancer get the switches associated to a LoadBalancer -func GetLogicalSwitchesForLoadBalancer(lb string) ([]string, error) { - out, _, err := util.RunOVNNbctl("--data=bare", "--no-heading", - "--columns=_uuid", "find", - "logical_switch", fmt.Sprintf("load_balancer{>=}%s", lb)) - if err != nil { - return nil, err + skipSNAT := "false" + // HACK(cdc) + // re-enable when BZ 1995326 is fixed + //if lb.Opts.SkipSNAT { + // skipSNAT = "true" + //} + + // Session affinity + // If enabled, then bucket flows by 3-tuple (proto, srcip, dstip) + // otherwise, use default ovn value + selectionFields := "[]" // empty set + if lb.Opts.Affinity { + selectionFields = "ip_src,ip_dst" } - return strings.Fields(out), nil -} -// GetLogicalRoutersForLoadBalancer get the routers associated to a LoadBalancer -func GetLogicalRoutersForLoadBalancer(lb string) ([]string, error) { - out, _, err := util.RunOVNNbctl("--data=bare", "--no-heading", - "--columns=name", "find", - "logical_router", fmt.Sprintf("load_balancer{>=}%s", lb)) - if err != nil { - return nil, err + // vipSet + vipSet := make([]string, 0, len(lb.Rules)) + for _, rule := range lb.Rules { + vipSet = append(vipSet, rule.nbctlString()) } - return strings.Fields(out), nil -} -// GetGRLogicalSwitchForLoadBalancer returns the external switch name if the load balancer is on a GR -func GetGRLogicalSwitchForLoadBalancer(lb string) (string, error) { - routers, err := GetLogicalRoutersForLoadBalancer(lb) - if err != nil { - return "", err - } - if len(routers) == 0 { - return "", nil + out := []string{ + "name=" + lb.Name, + "protocol=" + strings.ToLower(lb.Protocol), + "selection_fields=" + selectionFields, + "options:reject=" + reject, + "options:event=" + event, + "options:skip_snat=" + skipSNAT, + fmt.Sprintf(`vips={%s}`, strings.Join(vipSet, ",")), } - // if this is a GR we know the corresponding join and external switches, otherwise this is an unhandled - // case - for _, r := range routers { - if strings.HasPrefix(r, types.GWRouterPrefix) { - routerName := strings.TrimPrefix(r, types.GWRouterPrefix) - return types.ExternalSwitchPrefix + routerName, nil - } + for k, v := range lb.ExternalIDs { + out = append(out, "external_ids:"+k+"="+v) } - return "", fmt.Errorf("router detected with load balancer that is not a GR") + + // for unit testing - stable order + sort.Strings(out) + + return out } -// GenerateACLName generates a deterministic ACL name based on the load_balancer parameters -func GenerateACLName(lb string, sourceIP string, sourcePort int32) string { - aclName := fmt.Sprintf("%s-%s:%d", lb, sourceIP, sourcePort) - // ACL names are limited to 63 characters - if len(aclName) > 63 { - var ipPortLen int - srcPortStr := fmt.Sprintf("%d", sourcePort) - // Add the length of the IP (max 15 with periods, max 39 with colons), - // plus length of sourcePort (max 5 char), - // plus 1 for additional ':' to separate, - // plus 1 for '-' between lb and IP. - // With full IPv6 address and 5 char port, max ipPortLen is 62 - // With full IPv4 address and 5 char port, max ipPortLen is 24. - ipPortLen = len(sourceIP) + len(srcPortStr) + 1 + 1 - lbTrim := 63 - ipPortLen - // Shorten the Load Balancer name to allow full IP:port - tmpLb := lb[:lbTrim] - klog.Infof("Limiting ACL Name from %s to %s-%s:%d to keep under 63 characters", aclName, tmpLb, sourceIP, sourcePort) - aclName = fmt.Sprintf("%s-%s:%d", tmpLb, sourceIP, sourcePort) +// Returns a nbctl column update string for this rule +func (r *LBRule) nbctlString() string { + tgts := make([]string, 0, len(r.Targets)) + for _, tgt := range r.Targets { + tgts = append(tgts, tgt.String()) } - return aclName + + return fmt.Sprintf(`"%s"="%s"`, + r.Source.String(), + strings.Join(tgts, ",")) } -func GetWorkerLoadBalancer(node string, protocol kapi.Protocol) (string, error) { - var out string - var err error - if protocol == kapi.ProtocolTCP { - out, _, err = util.FindOVNLoadBalancer(types.WorkerLBTCP, node) - } else if protocol == kapi.ProtocolUDP { - out, _, err = util.FindOVNLoadBalancer(types.WorkerLBUDP, node) - } else if protocol == kapi.ProtocolSCTP { - out, _, err = util.FindOVNLoadBalancer(types.WorkerLBSCTP, node) +// DeleteLBs deletes all load balancer uuids supplied +// Note: this also automatically removes them from the switches and the routers :-) +func DeleteLBs(txn *util.NBTxn, uuids []string) error { + if len(uuids) == 0 { + return nil } + + cache, err := GetLBCache() if err != nil { - return "", err + return err } - if out == "" { - return "", fmt.Errorf("no %s load balancer found in the database for worker %s", protocol, node) + + commit := false + if txn == nil { + txn = util.NewNBTxn() + commit = true } - return out, nil -} + args := append([]string{"--if-exists", "destroy", "Load_Balancer"}, uuids...) -// GetWorkerLoadBalancers find TCP, SCTP, UDP load-balancers from worker -func GetWorkerLoadBalancers(node string) (string, string, string, error) { - lbTCP, stderr, err := util.FindOVNLoadBalancer(types.WorkerLBTCP, node) + _, _, err = txn.AddOrCommit(args) if err != nil { - return "", "", "", errors.Wrapf(err, "failed to get gateway router %q TCP "+ - "load balancer, stderr: %q", node, stderr) + return err } + if commit { + if _, _, err := txn.Commit(); err != nil { + return err + } + cache.update(nil, uuids) - lbUDP, stderr, err := util.FindOVNLoadBalancer(types.WorkerLBUDP, node) - if err != nil { - return "", "", "", errors.Wrapf(err, "failed to get gateway router %q UDP "+ - "load balancer, stderr: %q", node, stderr) } + return err +} + +type DeleteVIPEntry struct { + LBUUID string + VIPs []string // ip:string (or v6 equivalent) +} - lbSCTP, stderr, err := util.FindOVNLoadBalancer(types.WorkerLBSCTP, node) +// DeleteLoadBalancerVIPs removes VIPs from load-balancers in a single shot. +func DeleteLoadBalancerVIPs(toRemove []DeleteVIPEntry) error { + lbCache, err := GetLBCache() if err != nil { - return "", "", "", errors.Wrapf(err, "failed to get gateway router %q SCTP "+ - "load balancer, stderr: %q", node, stderr) + return err } - return lbTCP, lbUDP, lbSCTP, nil -} -// CreateLoadBalancerVIPs either creates or updates a set of load balancer VIPs mapping -// from SourcePort on each IP of a given address family in sourceIPs, to TargetPort on -// each IP of the same address family in TargetIPs -func CreateLoadBalancerVIPs(lb string, - sourceIPs []string, sourcePort int32, - targetIPs []string, targetPort int32) error { - klog.V(5).Infof("Creating lb with %s, [%v], %d, [%v], %d", lb, sourceIPs, sourcePort, targetIPs, targetPort) txn := util.NewNBTxn() - for _, sourceIP := range sourceIPs { - isIPv6 := utilnet.IsIPv6String(sourceIP) - - var targets []string - for _, targetIP := range targetIPs { - if utilnet.IsIPv6String(targetIP) == isIPv6 { - targets = append(targets, util.JoinHostPortInt32(targetIP, targetPort)) - } + + for _, entry := range toRemove { + if len(entry.VIPs) == 0 { + continue + } + + vipsStr := strings.Builder{} + // neat trick: leading spaces don't matter + for _, vip := range entry.VIPs { + vipsStr.WriteString(fmt.Sprintf(` "%s"`, vip)) } - vip := util.JoinHostPortInt32(sourceIP, sourcePort) - lbTarget := fmt.Sprintf(`vips:"%s"="%s"`, vip, strings.Join(targets, ",")) - request := []string{"set", "load_balancer", lb, lbTarget} - _, stderr, err := txn.AddOrCommit(request) + + _, _, err := txn.AddOrCommit([]string{ + "--if-exists", "remove", "load_balancer", entry.LBUUID, "vips", vipsStr.String(), + }) if err != nil { - return fmt.Errorf("unable to create load balancer: stderr: %s, err: %v", stderr, err) + return fmt.Errorf("failed to remove vips from load_balancer: %w", err) } } - _, stderr, err := txn.Commit() + + _, _, err = txn.Commit() if err != nil { - return fmt.Errorf("unable to create load balancer: stderr: %s, err: %v", stderr, err) + return fmt.Errorf("failed to remove vips from load_balancer: %w", err) } - return nil -} -type Entry struct { - LoadBalancer string - SourceIPS []string - SourcePort int32 - TargetIPs []string - TargetPort int32 -} + lbCache.removeVips(toRemove) -// BundleCreateLoadBalancerVIPs is the same as CreateLoadBalancerVIPs but batches multiple load balancer config -// together into a single transaction -// creates or updates a set of load balancer VIPs mapping -// from SourcePort on each IP of a given address family in sourceIPs, to TargetPort on -// each IP of the same address family in TargetIPs -func BundleCreateLoadBalancerVIPs(lbEntries []Entry) error { - txn := util.NewNBTxn() - for _, entry := range lbEntries { - for _, sourceIP := range entry.SourceIPS { - isIPv6 := utilnet.IsIPv6String(sourceIP) - var targets []string - for _, targetIP := range entry.TargetIPs { - if utilnet.IsIPv6String(targetIP) == isIPv6 { - targets = append(targets, util.JoinHostPortInt32(targetIP, entry.TargetPort)) - } - } - vip := util.JoinHostPortInt32(sourceIP, entry.SourcePort) - lbTarget := fmt.Sprintf(`vips:"%s"="%s"`, vip, strings.Join(targets, ",")) - request := []string{"set", "load_balancer", entry.LoadBalancer, lbTarget} - _, stderr, err := txn.AddOrCommit(request) - if err != nil { - return fmt.Errorf("unable to create load balancer bundle: stderr: %s, err: %v", stderr, err) - } - } - } - _, stderr, err := txn.Commit() - if err != nil { - return fmt.Errorf("unable to create load balancer bundle: stderr: %s, err: %v", stderr, err) - } return nil } diff --git a/go-controller/pkg/ovn/loadbalancer/loadbalancer_test.go b/go-controller/pkg/ovn/loadbalancer/loadbalancer_test.go deleted file mode 100644 index 3537a04476..0000000000 --- a/go-controller/pkg/ovn/loadbalancer/loadbalancer_test.go +++ /dev/null @@ -1,287 +0,0 @@ -package loadbalancer - -import ( - "fmt" - "reflect" - "testing" - - ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - kapi "k8s.io/api/core/v1" -) - -func TestGetOVNKubeLoadBalancer(t *testing.T) { - tests := []struct { - name string - protocol kapi.Protocol - ovnCmd ovntest.ExpectedCmd - want string - wantErr bool - }{ - { - name: "existing loadbalancer TCP", - protocol: kapi.ProtocolTCP, - ovnCmd: ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: "a08ea426-2288-11eb-a30b-a8a1590cda29", - }, - want: "a08ea426-2288-11eb-a30b-a8a1590cda29", - wantErr: false, - }, - { - name: "non existing loadbalancer UDP", - protocol: kapi.ProtocolUDP, - ovnCmd: ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-udp=yes", - Output: "", - }, - want: "", - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fexec := ovntest.NewLooseCompareFakeExec() - fexec.AddFakeCmd(&tt.ovnCmd) - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - - got, err := GetOVNKubeLoadBalancer(tt.protocol) - if (err != nil) != tt.wantErr { - t.Errorf("GetOVNKubeLoadBalancer() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("GetOVNKubeLoadBalancer() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGetLoadBalancerVIPs(t *testing.T) { - tests := []struct { - name string - loadBalancer string - ovnCmd ovntest.ExpectedCmd - want map[string]string - wantErr bool - }{ - { - name: "loadbalancer with VIPs", - loadBalancer: "my-lb", - ovnCmd: ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer my-lb vips", - Output: `{"10.96.0.10:53"="10.244.2.3:53,10.244.2.5:53", "10.96.0.10:9153"="10.244.2.3:9153,10.244.2.5:9153", "10.96.0.1:443"="172.19.0.3:6443"}`, - }, - want: map[string]string{ - "10.96.0.10:53": "10.244.2.3:53,10.244.2.5:53", - "10.96.0.10:9153": "10.244.2.3:9153,10.244.2.5:9153", - "10.96.0.1:443": "172.19.0.3:6443", - }, - wantErr: false, - }, - { - name: "empty load balancer", - loadBalancer: "my-lb", - ovnCmd: ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading get load_balancer my-lb vips", - Output: "", - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fexec := ovntest.NewLooseCompareFakeExec() - fexec.AddFakeCmd(&tt.ovnCmd) - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - got, err := GetLoadBalancerVIPs(tt.loadBalancer) - if (err != nil) != tt.wantErr { - t.Errorf("GetLoadBalancerVIPs() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("GetLoadBalancerVIPs() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestDeleteLoadBalancerVIP(t *testing.T) { - tests := []struct { - name string - loadBalancer string - vip string - ovnCmd ovntest.ExpectedCmd - wantErr bool - }{ - { - name: "loadbalancer with VIPs", - loadBalancer: "my-lb", - vip: "10.96.0.10:53", - ovnCmd: ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer my-lb vips "10.96.0.10:53"`, - Output: `{"10.96.0.10:53"="10.244.2.3:53,10.244.2.5:53", "10.96.0.10:9153"="10.244.2.3:9153,10.244.2.5:9153", "10.96.0.1:443"="172.19.0.3:6443"}`, - }, - wantErr: false, - }, - { - name: "load balancer and OVN error", - loadBalancer: "my-lb", - vip: "10.96.0.10:53", - ovnCmd: ovntest.ExpectedCmd{ - Cmd: `ovn-nbctl --timeout=15 --if-exists remove load_balancer my-lb vips "10.96.0.10:53"`, - Output: "", - Err: fmt.Errorf("error while removing ACL: sw1, from switches"), - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fexec := ovntest.NewLooseCompareFakeExec() - fexec.AddFakeCmd(&tt.ovnCmd) - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - txn := util.NewNBTxn() - err = DeleteLoadBalancerVIP(txn, tt.loadBalancer, tt.vip) - if err != nil { - t.Errorf("DeleteLoadBalancerVIP error: %v", err) - } - _, _, err = txn.Commit() - if (err != nil) != tt.wantErr { - t.Errorf("DeleteLoadBalancerVIP() error = %v, wantErr %v", err, tt.wantErr) - return - } - }) - } -} - -func TestUpdateLoadBalancer(t *testing.T) { - type args struct { - lb string - vip string - targets []string - } - tests := []struct { - name string - args args - ovnCmd ovntest.ExpectedCmd - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fexec := ovntest.NewLooseCompareFakeExec() - fexec.AddFakeCmd(&tt.ovnCmd) - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - if err := UpdateLoadBalancer(tt.args.lb, tt.args.vip, tt.args.targets); (err != nil) != tt.wantErr { - t.Errorf("UpdateLoadBalancer() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestGetLogicalSwitchesForLoadBalancer(t *testing.T) { - type args struct { - lb string - } - tests := []struct { - name string - args args - ovnCmd ovntest.ExpectedCmd - want []string - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fexec := ovntest.NewLooseCompareFakeExec() - fexec.AddFakeCmd(&tt.ovnCmd) - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - got, err := GetLogicalSwitchesForLoadBalancer(tt.args.lb) - if (err != nil) != tt.wantErr { - t.Errorf("GetLogicalSwitchesForLoadBalancer() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("GetLogicalSwitchesForLoadBalancer() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGetLogicalRoutersForLoadBalancer(t *testing.T) { - type args struct { - lb string - } - tests := []struct { - name string - args args - ovnCmd ovntest.ExpectedCmd - want []string - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fexec := ovntest.NewLooseCompareFakeExec() - fexec.AddFakeCmd(&tt.ovnCmd) - err := util.SetExec(fexec) - if err != nil { - t.Errorf("fexec error: %v", err) - } - got, err := GetLogicalRoutersForLoadBalancer(tt.args.lb) - if (err != nil) != tt.wantErr { - t.Errorf("GetLogicalRoutersForLoadBalancer() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("GetLogicalRoutersForLoadBalancer() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGetGRLogicalSwitchForLoadBalancer(t *testing.T) { - type args struct { - lb string - } - tests := []struct { - name string - args args - want string - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := GetGRLogicalSwitchForLoadBalancer(tt.args.lb) - if (err != nil) != tt.wantErr { - t.Errorf("GetGRLogicalSwitchForLoadBalancer() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("GetGRLogicalSwitchForLoadBalancer() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/go-controller/pkg/ovn/loadbalancer/types.go b/go-controller/pkg/ovn/loadbalancer/types.go new file mode 100644 index 0000000000..540f7e1f89 --- /dev/null +++ b/go-controller/pkg/ovn/loadbalancer/types.go @@ -0,0 +1,56 @@ +package loadbalancer + +import "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + +// LB is a desired or existing load_balancer configuration in OVN. +type LB struct { + Name string + UUID string + Protocol string // one of TCP, UDP, SCTP + ExternalIDs map[string]string + Opts LBOpts + + Rules []LBRule + + // the names of logical switches and routers that this LB should be attached to + Switches []string + Routers []string +} + +type LBOpts struct { + // if true, then enable unidling. Otherwise, generate reject + Unidling bool + + // If true, then enable per-client-IP affinity. + Affinity bool + + // If true, then disable SNAT entirely + SkipSNAT bool +} + +type Addr struct { + IP string + Port int32 +} + +type LBRule struct { + Source Addr + Targets []Addr +} + +// JoinJostsPort takes a list of IPs and a port and converts it to a list of Addrs +func JoinHostsPort(ips []string, port int32) []Addr { + out := make([]Addr, 0, len(ips)) + for _, ip := range ips { + out = append(out, Addr{IP: ip, Port: port}) + } + return out +} + +func (a *Addr) String() string { + return util.JoinHostPortInt32(a.IP, a.Port) +} + +func (a *Addr) Equals(b *Addr) bool { + return a.Port == b.Port && a.IP == b.IP +} diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index db12a42321..b0d44009be 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -12,7 +12,6 @@ import ( "time" goovn "github.com/ebay/go-ovn" - "github.com/pkg/errors" kapi "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -29,7 +28,6 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/informer" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/ipallocator" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/loadbalancer" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -442,46 +440,6 @@ func (oc *Controller) SetupMaster(masterNodeName string) error { } } - // Create load balancers - - // If we enable idling we have to set the option before creating the loadbalancers - if config.Kubernetes.OVNEmptyLbEvents { - _, _, err := util.RunOVNNbctl("set", "nb_global", ".", "options:controller_event=true") - if err != nil { - klog.Error("Unable to enable controller events. Unidling not possible") - return err - } - } - - // We have 3 load-balancers per protocol to implement the East-west traffic // - protocols := []kapi.Protocol{kapi.ProtocolTCP, kapi.ProtocolUDP} - if oc.SCTPSupport { - protocols = append(protocols, kapi.ProtocolSCTP) - } - // Create load-balancers for east-west traffic for each protocol UDP, TCP, SCTP - // and for Idling services if empty-lb-backends is enabled - lbExternalIds := []string{types.ClusterLBPrefix} - // If we enable idling we have to set the option before creating the loadbalancers - // and create the new set of loadbalancers. - if config.Kubernetes.OVNEmptyLbEvents { - _, _, err := util.RunOVNNbctl("set", "nb_global", ".", "options:controller_event=true") - if err != nil { - klog.Error("Unable to enable controller events. Unidling not possible") - return err - } - lbExternalIds = append(lbExternalIds, types.ClusterIdlingLBPrefix) - } - // Create the LoadBalancers if they don´t exist - for _, lbExternalID := range lbExternalIds { - for _, p := range protocols { - uuid, err := loadbalancer.CreateLoadBalancer(p, lbExternalID) - if err != nil { - return errors.Wrapf(err, "Failed to create OVN load balancer for protocol %s", p) - } - oc.clusterLBsUUIDs = append(oc.clusterLBsUUIDs, uuid) - } - } - // Initialize the OVNJoinSwitch switch IP manager // The OVNJoinSwitch will be allocated IP addresses in the range 100.64.0.0/16 or fd98::/64. oc.joinSwIPManager, err = initJoinLogicalSwitchIPManager() @@ -634,34 +592,10 @@ func (oc *Controller) syncGatewayLogicalNetwork(node *kapi.Node, l3GatewayConfig return fmt.Errorf("failed to allocate join switch port IP address for node %s: %v", node.Name, err) } - // OCP HACK - // GatewayModeLocal is only used if Local mode is specified and None shared gateway bridge is specified - // This is to allow local gateway mode without having to configure/use the shared gateway bridge - // See https://github.com/openshift/ovn-kubernetes/pull/281 drLRPIPs, _ := oc.joinSwIPManager.getJoinLRPCacheIPs(types.OVNClusterRouter) - if l3GatewayConfig.Mode == config.GatewayModeLocal { - err = gatewayInitMinimal(node.Name, l3GatewayConfig, oc.SCTPSupport) - if err != nil { - return fmt.Errorf("failed to init local gateway with no OVS bridge: %v", err) - } - // END OCP HACK - } else { - err = gatewayInit(node.Name, clusterSubnets, hostSubnets, l3GatewayConfig, oc.SCTPSupport, gwLRPIPs, drLRPIPs) - if err != nil { - return fmt.Errorf("failed to init shared interface gateway: %v", err) - } - } - - // Add cluster load balancers to GR for Host -> Cluster IP Service traffic - if config.Gateway.Mode != config.GatewayModeLocal { - gr := util.GetGatewayRouterFromNode(node.Name) - for _, clusterLB := range oc.clusterLBsUUIDs { - _, stderr, err := util.RunOVNNbctl("--may-exist", "lr-lb-add", gr, clusterLB) - if err != nil { - return fmt.Errorf("unable to add cluster LB: %s to %s, stderr: %q, error: %v", - clusterLB, gr, stderr, err) - } - } + err = gatewayInit(node.Name, clusterSubnets, hostSubnets, l3GatewayConfig, oc.SCTPSupport, gwLRPIPs, drLRPIPs) + if err != nil { + return fmt.Errorf("failed to init shared interface gateway: %v", err) } // in the case of shared gateway mode, we need to setup @@ -698,30 +632,6 @@ func (oc *Controller) syncGatewayLogicalNetwork(node *kapi.Node, l3GatewayConfig } } - if l3GatewayConfig.NodePortEnable { - gatewayRouter := types.GWRouterPrefix + node.Name - if physicalIPs, _ := oc.getGatewayPhysicalIPs(gatewayRouter); physicalIPs == nil { - return fmt.Errorf("gateway physical IP for node %q does not yet exist", node.Name) - } - // if new services controller run a full sync on all services - // services that have host network endpoints, are nodeport, external IP or ingress all have unique - // per-node load balancers. Since we cannot determine which services those are without significant parsing - // just sync all services - err = oc.svcController.RequestFullSync() - } else { - // nodePort disabled, delete gateway load balancers for this node. - gatewayRouter := util.GetGatewayRouterFromNode(node.Name) - for _, proto := range []kapi.Protocol{kapi.ProtocolTCP, kapi.ProtocolUDP, kapi.ProtocolSCTP} { - lbUUID, _ := oc.getGatewayLoadBalancer(gatewayRouter, proto) - if lbUUID != "" { - _, _, err := util.RunOVNNbctl("--if-exists", "destroy", "load_balancer", lbUUID) - if err != nil { - klog.Errorf("Failed to destroy %s load balancer for gateway %s: %v", proto, gatewayRouter, err) - } - } - } - } - return err } @@ -885,17 +795,6 @@ func (oc *Controller) ensureNodeLogicalNetwork(node *kapi.Node, hostSubnets []*n klog.Errorf(err.Error()) return err } - for i, loadBalancerUUID := range oc.clusterLBsUUIDs { - if i == 0 { - stdout, stderr, err = util.RunOVNNbctl("set", "logical_switch", nodeName, "load_balancer="+loadBalancerUUID) - } else { - stdout, stderr, err = util.RunOVNNbctl("add", "logical_switch", nodeName, "load_balancer", loadBalancerUUID) - } - if err != nil { - klog.Errorf("Failed to set logical switch %v's load balancer, stdout: %q, stderr: %q, error: %v", nodeName, stdout, stderr, err) - return err - } - } // Add the node to the logical switch cache return oc.lsManager.AddNode(nodeName, hostSubnets) diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go index cf4dbd7d34..748634054b 100644 --- a/go-controller/pkg/ovn/master_test.go +++ b/go-controller/pkg/ovn/master_test.go @@ -48,9 +48,6 @@ type tNode struct { DrLrpIP string PhysicalBridgeMAC string SystemID string - TCPLBUUID string - UDPLBUUID string - SCTPLBUUID string NodeSubnet string GWRouter string ClusterIPNet string @@ -125,28 +122,13 @@ func cleanupGateway(fexec *ovntest.FakeExec, nodeName string, nodeSubnet string, "ovn-nbctl --timeout=15 --if-exist lr-del " + types.GWRouterPrefix + nodeName, "ovn-nbctl --timeout=15 --if-exist ls-del " + types.ExternalSwitchPrefix + nodeName, }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:TCP_lb_gateway_router=" + types.GWRouterPrefix + nodeName, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:UDP_lb_gateway_router=" + types.GWRouterPrefix + nodeName, - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:SCTP_lb_gateway_router=" + types.GWRouterPrefix + nodeName, - Output: "", - }) cleanupPBRandNATRules(fexec, nodeName, []*net.IPNet{ovntest.MustParseIPNet(nodeSubnet)}) } -func defaultFakeExec(nodeSubnet, nodeName string, sctpSupport bool) (*ovntest.FakeExec, string, string, string) { +func defaultFakeExec(nodeSubnet, nodeName string, sctpSupport bool) *ovntest.FakeExec { const ( - tcpLBUUID string = "1a3dfc82-2749-4931-9190-c30e7c0ecea3" - udpLBUUID string = "6d3142fc-53e8-4ac1-88e6-46094a5a9957" - sctpLBUUID string = "0514c521-a120-4756-aec6-883fe5db7139" - mgmtMAC string = "01:02:03:04:05:06" + mgmtMAC string = "01:02:03:04:05:06" ) fexec := ovntest.NewLooseCompareFakeExec() @@ -173,32 +155,6 @@ func defaultFakeExec(nodeSubnet, nodeName string, sctpSupport bool) (*ovntest.Fa "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find ACL match=\"(ip4.mcast || mldv1 || mldv2 || " + ipv6DynamicMulticastMatch + ")\" action=drop external-ids:default-deny-policy-type=Ingress", "ovn-nbctl --timeout=15 --id=@acl create acl priority=" + types.DefaultMcastDenyPriority + " direction=" + types.DirectionToLPort + " log=false match=\"(ip4.mcast || mldv1 || mldv2 || " + ipv6DynamicMulticastMatch + ")\" action=drop external-ids:default-deny-policy-type=Ingress -- add port_group acls @acl", }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-tcp=yes", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:k8s-cluster-lb-tcp=yes protocol=tcp", - Output: tcpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:k8s-cluster-lb-udp=yes protocol=udp", - Output: udpLBUUID, - }) - if sctpSupport { - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:k8s-cluster-lb-sctp=yes protocol=sctp", - Output: sctpLBUUID, - }) - } - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-udp=yes", - Output: "", - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:k8s-cluster-lb-sctp=yes", - Output: "", - }) drSwitchPort := types.JoinSwitchToGWRouterPrefix + types.OVNClusterRouter drRouterPort := types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter joinSubnetV4 := ovntest.MustParseIPNet("100.64.0.1/16") @@ -238,15 +194,6 @@ func defaultFakeExec(nodeSubnet, nodeName string, sctpSupport bool) (*ovntest.Fa Output: fakeUUID + "\n", }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 set logical_switch " + nodeName + " load_balancer=" + tcpLBUUID, - "ovn-nbctl --timeout=15 add logical_switch " + nodeName + " load_balancer " + udpLBUUID, - }) - if sctpSupport { - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 add logical_switch " + nodeName + " load_balancer " + sctpLBUUID, - }) - } fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 -- --may-exist lsp-add " + nodeName + " " + types.K8sPrefix + nodeName + " -- lsp-set-type " + types.K8sPrefix + nodeName + " -- lsp-set-options " + types.K8sPrefix + nodeName + " -- lsp-set-addresses " + types.K8sPrefix + nodeName + " " + mgmtMAC + " " + nodeMgmtPortIP.String(), }) @@ -262,54 +209,7 @@ func defaultFakeExec(nodeSubnet, nodeName string, sctpSupport bool) (*ovntest.Fa "ovn-nbctl --timeout=15 -- --if-exists set logical_switch " + nodeName + " other-config:exclude_ips=" + hybridOverlayIP.String(), }) - return fexec, tcpLBUUID, udpLBUUID, sctpLBUUID -} - -func addNodeportLBs(fexec *ovntest.FakeExec, nodeName, tcpLBUUID, udpLBUUID, sctpLBUUID string) { - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBTCP + "=" + types.GWRouterPrefix + nodeName, - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBUDP + "=" + types.GWRouterPrefix + nodeName, - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.GatewayLBSCTP + "=" + types.GWRouterPrefix + nodeName, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBTCP + "=" + nodeName, - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBUDP + "=" + nodeName, - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=_uuid find load_balancer external_ids:" + types.WorkerLBSCTP + "=" + nodeName, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.GatewayLBTCP + "=" + types.GWRouterPrefix + nodeName + " protocol=tcp", - Output: tcpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.GatewayLBUDP + "=" + types.GWRouterPrefix + nodeName + " protocol=udp", - Output: udpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.GatewayLBSCTP + "=" + types.GWRouterPrefix + nodeName + " protocol=sctp", - Output: sctpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 set logical_router " + types.GWRouterPrefix + nodeName + " load_balancer=" + tcpLBUUID + - "," + udpLBUUID + "," + sctpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.WorkerLBTCP + "=" + nodeName + " protocol=tcp", - Output: tcpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.WorkerLBUDP + "=" + nodeName + " protocol=udp", - Output: udpLBUUID, - }) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 -- create load_balancer external_ids:" + types.WorkerLBSCTP + "=" + nodeName + " protocol=sctp", - Output: sctpLBUUID, - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 get logical_switch " + nodeName + " load_balancer", - "ovn-nbctl --timeout=15 ls-lb-add " + nodeName + " " + tcpLBUUID, - "ovn-nbctl --timeout=15 ls-lb-add " + nodeName + " " + udpLBUUID, - "ovn-nbctl --timeout=15 ls-lb-add " + nodeName + " " + sctpLBUUID, - }) + return fexec } func addNodeLogicalFlows(fexec *ovntest.FakeExec, node *tNode, clusterCIDR string, enableIPv6, sync bool) { @@ -330,11 +230,9 @@ func addNodeLogicalFlows(fexec *ovntest.FakeExec, node *tNode, clusterCIDR strin }) fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-nbctl --timeout=15 set logical_switch " + node.Name + " load_balancer=" + node.TCPLBUUID, - "ovn-nbctl --timeout=15 add logical_switch " + node.Name + " load_balancer " + node.UDPLBUUID, - "ovn-nbctl --timeout=15 add logical_switch " + node.Name + " load_balancer " + node.SCTPLBUUID, "ovn-nbctl --timeout=15 -- --may-exist lsp-add " + node.Name + " " + types.K8sPrefix + node.Name + " -- lsp-set-type " + types.K8sPrefix + node.Name + " -- lsp-set-options " + types.K8sPrefix + node.Name + " -- lsp-set-addresses " + types.K8sPrefix + node.Name + " " + node.NodeMgmtPortMAC + " " + node.NodeMgmtPortIP, }) + fexec.AddFakeCmd(&ovntest.ExpectedCmd{ Cmd: "ovn-nbctl --timeout=15 get logical_switch_port " + types.K8sPrefix + node.Name + " _uuid", Output: fakeUUID + "\n", @@ -363,7 +261,6 @@ func addNodeLogicalFlows(fexec *ovntest.FakeExec, node *tNode, clusterCIDR strin "ovn-nbctl --timeout=15 set logical_router " + node.GWRouter + " options:dynamic_neigh_routers=true", "ovn-nbctl --timeout=15 --may-exist lr-route-add " + node.GWRouter + " " + clusterCIDR + " " + node.DrLrpIP, }) - addNodeportLBs(fexec, node.Name, node.TCPLBUUID, node.UDPLBUUID, node.SCTPLBUUID) fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 --may-exist ls-add " + types.ExternalSwitchPrefix + node.Name, }) @@ -384,17 +281,10 @@ func addNodeLogicalFlows(fexec *ovntest.FakeExec, node *tNode, clusterCIDR strin fmt.Sprintf("ovn-nbctl --timeout=15 --columns _uuid --format=csv --no-headings find nat external_ip=\"%s\" type=snat logical_ip=\"%s\"", node.GatewayRouterIP, clusterCIDR), "ovn-nbctl --timeout=15 --if-exists lr-nat-del " + node.GWRouter + " snat " + clusterCIDR, "ovn-nbctl --timeout=15 lr-nat-add " + node.GWRouter + " snat " + node.GatewayRouterIP + " " + clusterCIDR, - "ovn-nbctl --timeout=15 --may-exist lr-lb-add " + node.GWRouter + " " + node.TCPLBUUID, - "ovn-nbctl --timeout=15 --may-exist lr-lb-add " + node.GWRouter + " " + node.UDPLBUUID, - "ovn-nbctl --timeout=15 --may-exist lr-lb-add " + node.GWRouter + " " + node.SCTPLBUUID, }) addPBRandNATRules(fexec, node.Name, node.NodeSubnet, node.GatewayRouterIP, node.NodeMgmtPortIP, node.NodeMgmtPortMAC) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 get logical_router " + types.GWRouterPrefix + node.Name + " external_ids:physical_ips", - Output: "169.254.33.2", - }) if sync { fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 -- --may-exist lr-add " + node.GWRouter + " -- set logical_router " + node.GWRouter + " options:chassis=" + node.SystemID + " external_ids:physical_ip=" + node.GatewayRouterIP + " external_ids:physical_ips=" + node.GatewayRouterIP, @@ -409,7 +299,6 @@ func addNodeLogicalFlows(fexec *ovntest.FakeExec, node *tNode, clusterCIDR strin "ovn-nbctl --timeout=15 set logical_router " + node.GWRouter + " options:dynamic_neigh_routers=true", "ovn-nbctl --timeout=15 --may-exist lr-route-add " + node.GWRouter + " " + clusterCIDR + " " + node.DrLrpIP, }) - addNodeportLBs(fexec, node.Name, node.TCPLBUUID, node.UDPLBUUID, node.SCTPLBUUID) fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 --may-exist ls-add " + types.ExternalSwitchPrefix + node.Name, }) @@ -432,9 +321,6 @@ func addNodeLogicalFlows(fexec *ovntest.FakeExec, node *tNode, clusterCIDR strin fmt.Sprintf("ovn-nbctl --timeout=15 --columns _uuid --format=csv --no-headings find nat external_ip=\"%s\" type=snat logical_ip=\"%s\"", node.GatewayRouterIP, clusterCIDR), "ovn-nbctl --timeout=15 --if-exists lr-nat-del " + node.GWRouter + " snat " + clusterCIDR, "ovn-nbctl --timeout=15 lr-nat-add " + node.GWRouter + " snat " + node.GatewayRouterIP + " " + clusterCIDR, - "ovn-nbctl --timeout=15 --may-exist lr-lb-add " + node.GWRouter + " " + node.TCPLBUUID, - "ovn-nbctl --timeout=15 --may-exist lr-lb-add " + node.GWRouter + " " + node.UDPLBUUID, - "ovn-nbctl --timeout=15 --may-exist lr-lb-add " + node.GWRouter + " " + node.SCTPLBUUID, }) } @@ -1179,9 +1065,6 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { DrLrpIP: "100.64.0.1", PhysicalBridgeMAC: "11:22:33:44:55:66", SystemID: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", - TCPLBUUID: "d2e858b2-cb5a-441b-a670-ed450f79a91f", - UDPLBUUID: "12832f14-eb0f-44d4-b8db-4cccbc73c792", - SCTPLBUUID: "0514c521-a120-4756-aec6-883fe5db7139", NodeSubnet: "10.1.1.0/24", GWRouter: types.GWRouterPrefix + "node1", GatewayRouterIPMask: "172.16.16.2/24", @@ -1261,11 +1144,6 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { addPBRandNATRules(fexec, node1.Name, node1.NodeSubnet, node1.GatewayRouterIP, node1.NodeMgmtPortIP, node1.NodeMgmtPortMAC) - fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 get logical_router " + types.GWRouterPrefix + node1.Name + " external_ids:physical_ips", - Output: "169.254.33.2", - }) - f, err = factory.NewMasterWatchFactory(fakeClient) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1279,7 +1157,6 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { record.NewFakeRecorder(0)) gomega.Expect(clusterController).NotTo(gomega.BeNil()) - clusterController.clusterLBsUUIDs = []string{node1.TCPLBUUID, node1.UDPLBUUID, node1.SCTPLBUUID} clusterController.SCTPSupport = true clusterController.joinSwIPManager, _ = initJoinLogicalSwitchIPManager() _, _ = clusterController.joinSwIPManager.ensureJoinLRPIPs(types.OVNClusterRouter) @@ -1290,10 +1167,11 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { clusterController.clusterRtrPortGroupUUID, err = createPortGroup(clusterController.ovnNBClient, clusterRtrPortGroupName, clusterRtrPortGroupName) clusterController.clusterPortGroupUUID, err = createPortGroup(clusterController.ovnNBClient, clusterPortGroupName, clusterPortGroupName) - clusterController.StartServiceController(wg, false) // Let the real code run and ensure OVN database sync clusterController.WatchNodes() + clusterController.StartServiceController(wg, false) + subnet := ovntest.MustParseIPNet(node1.NodeSubnet) err = clusterController.syncGatewayLogicalNetwork(updatedNode, l3GatewayConfig, []*net.IPNet{subnet}, hostAddrs) gomega.Expect(fexec.CalledMatchesExpected()).To(gomega.BeTrue(), fexec.ErrorDesc) diff --git a/go-controller/pkg/ovn/namespace_test.go b/go-controller/pkg/ovn/namespace_test.go index fbf806fd59..1531142047 100644 --- a/go-controller/pkg/ovn/namespace_test.go +++ b/go-controller/pkg/ovn/namespace_test.go @@ -2,9 +2,10 @@ package ovn import ( "context" + "net" + "github.com/urfave/cli/v2" "k8s.io/apimachinery/pkg/util/sets" - "net" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" egressfirewallfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/clientset/versioned/fake" @@ -159,9 +160,6 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { DrLrpIP: "100.64.0.1", PhysicalBridgeMAC: "11:22:33:44:55:66", SystemID: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", - TCPLBUUID: "d2e858b2-cb5a-441b-a670-ed450f79a91f", - UDPLBUUID: "12832f14-eb0f-44d4-b8db-4cccbc73c792", - SCTPLBUUID: "0514c521-a120-4756-aec6-883fe5db7139", NodeSubnet: "10.1.1.0/24", GWRouter: ovntypes.GWRouterPrefix + "node1", GatewayRouterIPMask: "172.16.16.2/24", @@ -245,7 +243,6 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { fakeOvn.fakeClient = fakeClient fakeOvn.init() fakeOvn.controller.multicastSupport = false - fakeOvn.controller.clusterLBsUUIDs = []string{node1.TCPLBUUID, node1.UDPLBUUID, node1.SCTPLBUUID} _, clusterNetwork, err := net.ParseCIDR(clusterCIDR) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -268,10 +265,10 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { fakeOvn.controller.WatchNamespaces() fakeOvn.asf.EventuallyExpectEmptyAddressSet(hostNetworkNamespace) - fakeOvn.controller.StartServiceController(fakeOvn.wg, false) - fakeOvn.controller.WatchNodes() + fakeOvn.controller.StartServiceController(fakeOvn.wg, false) + gomega.Expect(fexec.CalledMatchesExpected()).To(gomega.BeTrue(), fexec.ErrorDesc) // check the namespace again and ensure the address set diff --git a/go-controller/pkg/ovn/ovn.go b/go-controller/pkg/ovn/ovn.go index f757d217c8..b70f3c02c4 100644 --- a/go-controller/pkg/ovn/ovn.go +++ b/go-controller/pkg/ovn/ovn.go @@ -123,9 +123,7 @@ type Controller struct { hoMaster *hocontroller.MasterController - // All the uuid related to global load balancers - clusterLBsUUIDs []string - SCTPSupport bool + SCTPSupport bool // For TCP, UDP, and SCTP type traffic, cache OVN load-balancers used for the // cluster's east-west traffic. @@ -295,7 +293,7 @@ func NewOvnController(ovnClient *util.OVNClientset, wf *factory.WatchFactory, st ovnSBClient: ovnSBClient, nbClient: libovsdbOvnNBClient, sbClient: libovsdbOvnSBClient, - clusterLBsUUIDs: make([]string, 0), + svcController: newServiceController(ovnClient.KubeClient, libovsdbOvnNBClient, stopChan), } } @@ -309,16 +307,16 @@ func (oc *Controller) Run(wg *sync.WaitGroup, nodeName string) error { // dependencies, and WatchNodes() depends on it oc.WatchNamespaces() - // Services must be started before nodes for handling new node's service sync - if err := oc.StartServiceController(wg, true); err != nil { - return err - } - // WatchNodes must be started next because it creates the node switch // which most other watches depend on. // https://github.com/ovn-org/ovn-kubernetes/pull/859 oc.WatchNodes() + // Services should be started after nodes to prevent LB churn + if err := oc.StartServiceController(wg, true); err != nil { + return err + } + oc.WatchPods() // WatchNetworkPolicy depends on WatchPods and WatchNamespaces @@ -879,6 +877,7 @@ func (oc *Controller) WatchNamespaces() { klog.Infof("Bootstrapping existing namespaces and cleaning stale namespaces took %v", time.Since(start)) } +// syncNodeGateway ensures a node's gateway router is configured func (oc *Controller) syncNodeGateway(node *kapi.Node, hostSubnets []*net.IPNet) error { l3GatewayConfig, err := util.ParseNodeL3GatewayAnnotation(node) if err != nil { @@ -1140,49 +1139,48 @@ func shouldUpdate(node, oldNode *kapi.Node) (bool, error) { return true, nil } -func (oc *Controller) newServiceFactory() (informers.SharedInformerFactory, error) { +func newServiceController(client clientset.Interface, nbClient libovsdbclient.Client, stopChan <-chan struct{}) *svccontroller.Controller { // Create our own informers to start compartmentalizing the code // filter server side the things we don't care about noProxyName, err := labels.NewRequirement("service.kubernetes.io/service-proxy-name", selection.DoesNotExist, nil) if err != nil { - return nil, err + panic(err) } noHeadlessEndpoints, err := labels.NewRequirement(kapi.IsHeadlessService, selection.DoesNotExist, nil) if err != nil { - return nil, err + panic(err) } labelSelector := labels.NewSelector() labelSelector = labelSelector.Add(*noProxyName, *noHeadlessEndpoints) - return informers.NewSharedInformerFactoryWithOptions(oc.client, 0, + svcFactory := informers.NewSharedInformerFactoryWithOptions(client, 0, informers.WithTweakListOptions(func(options *metav1.ListOptions) { options.LabelSelector = labelSelector.String() - })), nil -} + })) -func (oc *Controller) StartServiceController(wg *sync.WaitGroup, runRepair bool) error { - klog.Infof("Starting OVN Service Controller: Using Endpoint Slices") - svcFactory, err := oc.newServiceFactory() - if err != nil { - return err - } - - oc.svcController = svccontroller.NewController( - oc.client, - oc.nbClient, + controller := svccontroller.NewController( + client, + nbClient, svcFactory.Core().V1().Services(), svcFactory.Discovery().V1beta1().EndpointSlices(), - oc.clusterPortGroupUUID, + svcFactory.Core().V1().Nodes(), ) - svcFactory.Start(oc.stopChan) + + svcFactory.Start(stopChan) + + return controller +} + +func (oc *Controller) StartServiceController(wg *sync.WaitGroup, runRepair bool) error { + klog.Infof("Starting OVN Service Controller: Using Endpoint Slices") wg.Add(1) go func() { defer wg.Done() // use 5 workers like most of the kubernetes controllers in the // kubernetes controller-manager - err := oc.svcController.Run(5, oc.stopChan, runRepair) + err := oc.svcController.Run(5, oc.stopChan, runRepair, oc.clusterPortGroupUUID) if err != nil { klog.Errorf("Error running OVN Kubernetes Services controller: %v", err) } diff --git a/go-controller/pkg/types/const.go b/go-controller/pkg/types/const.go index da8ebfe702..10f687c19d 100644 --- a/go-controller/pkg/types/const.go +++ b/go-controller/pkg/types/const.go @@ -88,20 +88,6 @@ const ( OvnACLLoggingMeter = "acl-logging" - // LoadBalancer External Names - ClusterLBTCP = "k8s-cluster-lb-tcp" - ClusterLBUDP = "k8s-cluster-lb-udp" - ClusterLBSCTP = "k8s-cluster-lb-sctp" - ClusterLBPrefix = "k8s-cluster-lb" - ClusterIdlingLBPrefix = "k8s-idling-lb" - WorkerLBPrefix = "k8s-worker-lb" - WorkerLBTCP = WorkerLBPrefix + "-tcp" - WorkerLBUDP = WorkerLBPrefix + "-udp" - WorkerLBSCTP = WorkerLBPrefix + "-sctp" - GatewayLBTCP = "TCP_lb_gateway_router" - GatewayLBUDP = "UDP_lb_gateway_router" - GatewayLBSCTP = "SCTP_lb_gateway_router" - // OVN-K8S Topology Versions OvnSingleJoinSwitchTopoVersion = 1 OvnNamespacedDenyPGTopoVersion = 2 diff --git a/go-controller/pkg/util/kube.go b/go-controller/pkg/util/kube.go index 8bbce7fed1..696e566cff 100644 --- a/go-controller/pkg/util/kube.go +++ b/go-controller/pkg/util/kube.go @@ -10,6 +10,9 @@ import ( kapi "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -20,12 +23,12 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/cert" "k8s.io/klog/v2" - utilnet "k8s.io/utils/net" egressfirewallclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/clientset/versioned" egressipclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/clientset/versioned" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" + cnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" ) @@ -224,9 +227,9 @@ const ( // // Note that the changes below is based on following assumptions, which is true today. // - a pod's default network is OVN managed -func GetPodNetSelAnnotation(pod *kapi.Pod, netAttachAnnot string) ([]*types.NetworkSelectionElement, error) { +func GetPodNetSelAnnotation(pod *kapi.Pod, netAttachAnnot string) ([]*cnitypes.NetworkSelectionElement, error) { var networkAnnotation string - var networks []*types.NetworkSelectionElement + var networks []*cnitypes.NetworkSelectionElement networkAnnotation = pod.Annotations[netAttachAnnot] if networkAnnotation == "" { @@ -274,27 +277,24 @@ func UseEndpointSlices(kubeClient kubernetes.Interface) bool { } type LbEndpoints struct { - IPs []string - Port int32 + V4IPs []string + V6IPs []string + Port int32 } // GetLbEndpoints return the endpoints that belong to the IPFamily as a slice of IPs -func GetLbEndpoints(slices []*discovery.EndpointSlice, svcPort kapi.ServicePort, family kapi.IPFamily) LbEndpoints { - epsSet := sets.NewString() - lbEps := LbEndpoints{[]string{}, 0} +func GetLbEndpoints(slices []*discovery.EndpointSlice, svcPort kapi.ServicePort) LbEndpoints { + v4ips := sets.NewString() + v6ips := sets.NewString() + + out := LbEndpoints{} // return an empty object so the caller don't have to check for nil and can use it as an iterator if len(slices) == 0 { - return lbEps + return out } for _, slice := range slices { - klog.V(4).Infof("Getting endpoints for slice %s", slice.Name) - // Only return addresses that belong to the requested IP family - if slice.AddressType != discovery.AddressType(family) { - klog.V(4).Infof("Slice %s with different IP Family endpoints, requested: %s received: %s", - slice.Name, slice.AddressType, family) - continue - } + klog.V(4).Infof("Getting endpoints for slice %s/%s", slice.Namespace, slice.Name) // build the list of endpoints in the slice for _, port := range slice.Ports { @@ -313,7 +313,7 @@ func GetLbEndpoints(slices []*discovery.EndpointSlice, svcPort kapi.ServicePort, continue } - lbEps.Port = *port.Port + out.Port = *port.Port for _, endpoint := range slice.Endpoints { // Skip endpoints that are not ready if endpoint.Conditions.Ready != nil && !*endpoint.Conditions.Ready { @@ -322,38 +322,49 @@ func GetLbEndpoints(slices []*discovery.EndpointSlice, svcPort kapi.ServicePort, } for _, ip := range endpoint.Addresses { klog.V(4).Infof("Adding slice %s endpoints: %v, port: %d", slice.Name, endpoint.Addresses, *port.Port) - epsSet.Insert(ip) + switch slice.AddressType { + case discovery.AddressTypeIPv4: + v4ips.Insert(ip) + case discovery.AddressTypeIPv6: + v6ips.Insert(ip) + default: + klog.V(5).Infof("Skipping FQDN slice %s/%s", slice.Namespace, slice.Name) + } } } } } - lbEps.IPs = epsSet.List() - klog.V(4).Infof("LB Endpoints for %s are: %v on port: %d", slices[0].Labels[discovery.LabelServiceName], - lbEps.IPs, lbEps.Port) - return lbEps + out.V4IPs = v4ips.List() + out.V6IPs = v6ips.List() + klog.V(4).Infof("LB Endpoints for %s/%s are: %v / %v on port: %d", + slices[0].Namespace, slices[0].Labels[discovery.LabelServiceName], + out.V4IPs, out.V6IPs, out.Port) + return out } -// HasValidEndpoint returns true if at least one valid endpoint is contained in the given -// slices -func HasValidEndpoint(service *kapi.Service, slices []*discovery.EndpointSlice) bool { - if slices == nil { - return false - } - if len(slices) == 0 { - return false +type K8sObject interface { + metav1.Object + k8sruntime.Object +} + +func ExternalIDsForObject(obj K8sObject) map[string]string { + gk := obj.GetObjectKind().GroupVersionKind().GroupKind() + nsn := k8stypes.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), } - for _, ip := range GetClusterIPs(service) { - family := kapi.IPv4Protocol - if utilnet.IsIPv6String(ip) { - family = kapi.IPv6Protocol - } - for _, svcPort := range service.Spec.Ports { - eps := GetLbEndpoints(slices, svcPort, family) - if len(eps.IPs) > 0 { - return true - } + + if gk.String() == "" { + kinds, _, err := scheme.Scheme.ObjectKinds(obj) + if err != nil || len(kinds) == 0 || len(kinds) > 1 { + klog.Warningf("BUG: object has no / ambiguous GVK: %#v, err", obj, err) } + gk = kinds[0].GroupKind() + } + + return map[string]string{ + types.OvnK8sPrefix + "/owner": nsn.String(), + types.OvnK8sPrefix + "/kind": gk.String(), } - return false } diff --git a/go-controller/pkg/util/kube_test.go b/go-controller/pkg/util/kube_test.go index 1ffad945c7..4633d3630c 100644 --- a/go-controller/pkg/util/kube_test.go +++ b/go-controller/pkg/util/kube_test.go @@ -3,7 +3,6 @@ package util import ( "context" "fmt" - "reflect" "testing" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" @@ -458,7 +457,6 @@ func Test_getLbEndpoints(t *testing.T) { type args struct { slices []*discovery.EndpointSlice svcPort v1.ServicePort - family v1.IPFamily } tests := []struct { name string @@ -474,9 +472,8 @@ func Test_getLbEndpoints(t *testing.T) { TargetPort: intstr.FromInt(80), Protocol: v1.ProtocolTCP, }, - family: v1.IPv4Protocol, }, - want: LbEndpoints{[]string{}, 0}, + want: LbEndpoints{}, }, { name: "slices with endpoints", @@ -511,9 +508,8 @@ func Test_getLbEndpoints(t *testing.T) { TargetPort: intstr.FromInt(80), Protocol: v1.ProtocolTCP, }, - family: v1.IPv4Protocol, }, - want: LbEndpoints{[]string{"10.0.0.2"}, 80}, + want: LbEndpoints{[]string{"10.0.0.2"}, []string{}, 80}, }, { name: "slices with different port name", @@ -548,9 +544,8 @@ func Test_getLbEndpoints(t *testing.T) { TargetPort: intstr.FromInt(80), Protocol: v1.ProtocolTCP, }, - family: v1.IPv4Protocol, }, - want: LbEndpoints{[]string{}, 0}, + want: LbEndpoints{[]string{}, []string{}, 0}, }, { name: "slices and service without port name", @@ -583,9 +578,8 @@ func Test_getLbEndpoints(t *testing.T) { TargetPort: intstr.FromInt(80), Protocol: v1.ProtocolTCP, }, - family: v1.IPv4Protocol, }, - want: LbEndpoints{[]string{"10.0.0.2"}, 8080}, + want: LbEndpoints{[]string{"10.0.0.2"}, []string{}, 8080}, }, { name: "slices with different IP family", @@ -620,9 +614,8 @@ func Test_getLbEndpoints(t *testing.T) { TargetPort: intstr.FromInt(80), Protocol: v1.ProtocolTCP, }, - family: v1.IPv4Protocol, }, - want: LbEndpoints{[]string{}, 0}, + want: LbEndpoints{[]string{}, []string{"2001:db2::2"}, 80}, }, { name: "multiples slices with duplicate endpoints", @@ -680,16 +673,14 @@ func Test_getLbEndpoints(t *testing.T) { TargetPort: intstr.FromInt(80), Protocol: v1.ProtocolTCP, }, - family: v1.IPv4Protocol, }, - want: LbEndpoints{[]string{"10.0.0.2", "10.1.1.2", "10.2.2.2"}, 80}, + want: LbEndpoints{[]string{"10.0.0.2", "10.1.1.2", "10.2.2.2"}, []string{}, 80}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := GetLbEndpoints(tt.args.slices, tt.args.svcPort, tt.args.family); !reflect.DeepEqual(got, tt.want) { - t.Errorf("getLbEndpoints() = %v, want %v", got, tt.want) - } + got := GetLbEndpoints(tt.args.slices, tt.args.svcPort) + assert.Equal(t, tt.want, got) }) } } @@ -724,3 +715,36 @@ func TestPodScheduled(t *testing.T) { }) } } + +func TestExternalIDsForObject(t *testing.T) { + assert.Equal(t, + ExternalIDsForObject(&v1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-ab23", + Namespace: "ns", + Labels: map[string]string{discovery.LabelServiceName: "svc"}, + }, + }), + map[string]string{ + "k8s.ovn.org/kind": "Service", + "k8s.ovn.org/owner": "ns/svc-ab23", + }) + + assert.Equal(t, + ExternalIDsForObject(&v1.Service{ + // also handle no TypeMeta, which can happen. + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-ab23", + Namespace: "ns", + Labels: map[string]string{discovery.LabelServiceName: "svc"}, + }, + }), + map[string]string{ + "k8s.ovn.org/kind": "Service", + "k8s.ovn.org/owner": "ns/svc-ab23", + }) +} diff --git a/go-controller/pkg/util/ovs.go b/go-controller/pkg/util/ovs.go index f9626c7898..98e8591d77 100644 --- a/go-controller/pkg/util/ovs.go +++ b/go-controller/pkg/util/ovs.go @@ -2,9 +2,9 @@ package util import ( "bytes" + "encoding/csv" "encoding/json" "fmt" - "github.com/pkg/errors" "math" "os" "os/exec" @@ -17,6 +17,8 @@ import ( "time" "unicode" + "github.com/pkg/errors" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -525,13 +527,40 @@ func RunOVNNbctlUnix(args ...string) (string, string, error) { // RunOVNNbctlWithTimeout runs command via ovn-nbctl with a specific timeout func RunOVNNbctlWithTimeout(timeout int, args ...string) (string, string, error) { + stdout, stderr, err := RunOVNNbctlRawOutput(timeout, args...) + return strings.Trim(strings.TrimSpace(stdout), "\""), stderr, err +} + +// RunOVNNbctlRawOutput returns the output with no trimming or other string manipulation +func RunOVNNbctlRawOutput(timeout int, args ...string) (string, string, error) { cmdArgs, envVars := getNbctlArgsAndEnv(timeout, args...) start := time.Now() stdout, stderr, err := runOVNretry(runner.nbctlPath, envVars, cmdArgs...) if MetricOvnCliLatency != nil { MetricOvnCliLatency.WithLabelValues("ovn-nbctl").Observe(time.Since(start).Seconds()) } - return strings.Trim(strings.TrimSpace(stdout.String()), "\""), stderr.String(), err + return stdout.String(), stderr.String(), err +} + +// RunOVNNbctlCSV runs an nbctl command that results in CSV output, parses the rows returned, +// and returns the records +func RunOVNNbctlCSV(args []string) ([][]string, error) { + args = append([]string{"--no-heading", "--format=csv"}, args...) + + stdout, _, err := RunOVNNbctlRawOutput(15, args...) + if err != nil { + return nil, err + } + if len(stdout) == 0 { + return nil, nil + } + + r := csv.NewReader(strings.NewReader(stdout)) + records, err := r.ReadAll() + if err != nil { + return nil, fmt.Errorf("failed to parse nbctl CSV response: %w", err) + } + return records, nil } // RunOVNNbctl runs a command via ovn-nbctl. diff --git a/go-controller/pkg/util/util.go b/go-controller/pkg/util/util.go index 3b3c43d0ad..c85046e5a9 100644 --- a/go-controller/pkg/util/util.go +++ b/go-controller/pkg/util/util.go @@ -225,3 +225,40 @@ func UpdateIPsSlice(s, oldIPs, newIPs []string) []string { } return n } + +// FilterIPsSlice will filter a list of IPs by a list of CIDRs. By default, +// it will *remove* all IPs that match filter, unless keep is true. +// +// It is dual-stack aware. +func FilterIPsSlice(s []string, filter []net.IPNet, keep bool) []string { + out := make([]string, 0, len(s)) +ipLoop: + for _, ipStr := range s { + ip := net.ParseIP(ipStr) + is4 := ip.To4() != nil + + for _, cidr := range filter { + if is4 && cidr.IP.To4() != nil && cidr.Contains(ip) { + if keep { + out = append(out, ipStr) + continue ipLoop + } else { + continue ipLoop + } + } + if !is4 && cidr.IP.To4() == nil && cidr.Contains(ip) { + if keep { + out = append(out, ipStr) + continue ipLoop + } else { + continue ipLoop + } + } + } + if !keep { // discard mode, and nothing matched. + out = append(out, ipStr) + } + } + + return out +} diff --git a/go-controller/pkg/util/util_unit_test.go b/go-controller/pkg/util/util_unit_test.go index 6ee533ccaf..c77ba67b60 100644 --- a/go-controller/pkg/util/util_unit_test.go +++ b/go-controller/pkg/util/util_unit_test.go @@ -3,12 +3,14 @@ package util import ( "bytes" "fmt" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" - ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" "net" "reflect" + "strconv" "testing" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + mock_k8s_io_utils_exec "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/utils/exec" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/mocks" "github.com/stretchr/testify/assert" @@ -300,3 +302,71 @@ func TestUpdateIPsSlice(t *testing.T) { }) } } + +func TestFilterIPsSlice(t *testing.T) { + + var tests = []struct { + s, cidrs []string + keep bool + want []string + }{ + { + s: []string{"1.0.0.1", "2.0.0.1", "2001::1", "2002::1"}, + cidrs: []string{"1.0.0.0/24"}, + keep: true, + want: []string{"1.0.0.1"}, + }, + { + s: []string{"1.0.0.1", "2.0.0.1", "2001::1", "2002::1"}, + cidrs: []string{"1.0.0.0/24"}, + keep: false, + want: []string{"2.0.0.1", "2001::1", "2002::1"}, + }, + { + s: []string{"1.0.0.1", "2.0.0.1", "2001::1", "2002::1"}, + cidrs: []string{"2001::/64"}, + keep: true, + want: []string{"2001::1"}, + }, + { + s: []string{"1.0.0.1", "2.0.0.1", "2001::1", "2002::1"}, + cidrs: []string{"2001::/64"}, + keep: false, + want: []string{"1.0.0.1", "2.0.0.1", "2002::1"}, + }, + { + s: []string{"1.0.0.1", "2.0.0.1", "2001::1", "2002::1"}, + cidrs: []string{"1.0.0.0/24", "2001::/64", "3.0.0.0/24"}, + keep: false, + want: []string{"2.0.0.1", "2002::1"}, + }, + { + s: []string{"1.0.0.1", "2.0.0.1", "2001::1", "2002::1"}, + cidrs: []string{"1.0.0.0/24", "2001::/64", "3.0.0.0/24"}, + keep: true, + want: []string{"1.0.0.1", "2001::1"}, + }, + { + s: []string{"1.0.0.1", "2.0.0.1", "2001::1", "2002::1"}, + cidrs: []string{"1.0.0.0/24", "0.0.0.0/0"}, + keep: true, + want: []string{"1.0.0.1", "2.0.0.1"}, + }, + } + + for i, tc := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + cidrs := []net.IPNet{} + for _, cidr := range tc.cidrs { + _, n, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatal(err) + } + cidrs = append(cidrs, *n) + } + + actual := FilterIPsSlice(tc.s, cidrs, tc.keep) + assert.Equal(t, tc.want, actual) + }) + } +} diff --git a/test/scripts/e2e-kind.sh b/test/scripts/e2e-kind.sh index b3d6825d66..8a787398eb 100755 --- a/test/scripts/e2e-kind.sh +++ b/test/scripts/e2e-kind.sh @@ -26,8 +26,8 @@ should have ipv4 and ipv6 node podCIDRs kube-proxy should set TCP CLOSE_WAIT timeout -# TO BE IMPLEMENTED: https://github.com/ovn-org/ovn-kubernetes/issues/819 -Services.+session affinity +# not implemented - OVN doesn't support time +should have session affinity timeout work # NOT IMPLEMENTED; SEE DISCUSSION IN https://github.com/ovn-org/ovn-kubernetes/pull/1225 named port.+\[Feature:NetworkPolicy\]