diff --git a/go-controller/hybrid-overlay/pkg/controller/ho_node_windows.go b/go-controller/hybrid-overlay/pkg/controller/ho_node_windows.go index 5de9b75391..339bc289f7 100644 --- a/go-controller/hybrid-overlay/pkg/controller/ho_node_windows.go +++ b/go-controller/hybrid-overlay/pkg/controller/ho_node_windows.go @@ -57,7 +57,7 @@ func newNodeController(kube kube.Interface, "UDP port. Please make sure you install all the KB updates on your system.") } - node, err := kube.GetNode(nodeName) + node, err := kube.GetNodeForWindows(nodeName) if err != nil { return nil, err } @@ -345,7 +345,7 @@ func (n *NodeController) initSelf(node *corev1.Node, nodeSubnet *net.IPNet) erro } // Add existing nodes - nodes, err := n.kube.GetNodes() + nodes, err := n.kube.GetNodesForWindows() if err != nil { return fmt.Errorf("error in initializing/fetching nodes: %v", err) } @@ -370,7 +370,7 @@ func (n *NodeController) uninitSelf(node *corev1.Node) error { networkName, n.networkID, node.Name) // Remove existing nodes - nodes, err := n.kube.GetNodes() + nodes, err := n.kube.GetNodesForWindows() if err != nil { return fmt.Errorf("failed to get nodes: %v", err) } diff --git a/go-controller/hybrid-overlay/pkg/controller/ovn_node_linux.go b/go-controller/hybrid-overlay/pkg/controller/ovn_node_linux.go index 3c74239db0..df8a9559c8 100644 --- a/go-controller/hybrid-overlay/pkg/controller/ovn_node_linux.go +++ b/go-controller/hybrid-overlay/pkg/controller/ovn_node_linux.go @@ -261,7 +261,7 @@ func (n *NodeController) AddNode(node *corev1.Node) error { } else { // Make sure the local node has been initialized before adding a hybridOverlay remote node if atomic.LoadUint32(n.initState) < hotypes.DistributedRouterInitialized { - localNode, err := n.kube.GetNode(n.nodeName) + localNode, err := n.nodeLister.Get(n.nodeName) if err != nil { return fmt.Errorf("cannot get local node: %s: %w", n.nodeName, err) } diff --git a/go-controller/pkg/clustermanager/network_cluster_controller.go b/go-controller/pkg/clustermanager/network_cluster_controller.go index fde745ac00..ef2ac665ae 100644 --- a/go-controller/pkg/clustermanager/network_cluster_controller.go +++ b/go-controller/pkg/clustermanager/network_cluster_controller.go @@ -475,7 +475,7 @@ func (ncc *networkClusterController) Reconcile(netInfo util.NetInfo) error { klog.Errorf("Failed to reconcile network %s: %v", ncc.GetNetworkName(), err) } if reconcilePendingPods && ncc.retryPods != nil { - if err := objretry.RequeuePendingPods(ncc.kube, ncc.GetNetInfo(), ncc.retryPods); err != nil { + if err := objretry.RequeuePendingPods(ncc.watchFactory, ncc.GetNetInfo(), ncc.retryPods); err != nil { klog.Errorf("Failed to requeue pending pods for network %s: %v", ncc.GetNetworkName(), err) } } @@ -576,7 +576,7 @@ func (h *networkClusterControllerEventHandler) UpdateResource(oldObj, newObj int // 1. we missed an add event (bug in kapi informer code) // 2. a user removed the annotation on the node // Either way to play it safe for now do a partial json unmarshal check - if !nodeFailed && util.NoHostSubnet(oldNode) != util.NoHostSubnet(newNode) && !h.ncc.nodeAllocator.NeedsNodeAllocation(newNode) { + if !nodeFailed && util.NoHostSubnet(oldNode) == util.NoHostSubnet(newNode) && !h.ncc.nodeAllocator.NeedsNodeAllocation(newNode) { // no other node updates would require us to reconcile again return nil } diff --git a/go-controller/pkg/clustermanager/node/node_allocator.go b/go-controller/pkg/clustermanager/node/node_allocator.go index 63593618b2..83c3d80fde 100644 --- a/go-controller/pkg/clustermanager/node/node_allocator.go +++ b/go-controller/pkg/clustermanager/node/node_allocator.go @@ -195,27 +195,24 @@ func (na *NodeAllocator) NeedsNodeAllocation(node *corev1.Node) bool { } // ovn node check - // allocation is all or nothing, so if one field was allocated from: - // nodeSubnets, joinSubnet, layer 2 tunnel id, then all of them were if na.hasNodeSubnetAllocation() { - if util.HasNodeHostSubnetAnnotation(node, na.netInfo.GetNetworkName()) { - return false + if !util.HasNodeHostSubnetAnnotation(node, na.netInfo.GetNetworkName()) { + return true } } - if na.hasJoinSubnetAllocation() { - if util.HasNodeGatewayRouterJoinNetwork(node, na.netInfo.GetNetworkName()) { - return false + if !util.HasNodeGatewayRouterJoinNetwork(node, na.netInfo.GetNetworkName()) { + return true } } if util.IsNetworkSegmentationSupportEnabled() && na.netInfo.IsPrimaryNetwork() && util.DoesNetworkRequireTunnelIDs(na.netInfo) { - if util.HasUDNLayer2NodeGRLRPTunnelID(node, na.netInfo.GetNetworkName()) { - return false + if !util.HasUDNLayer2NodeGRLRPTunnelID(node, na.netInfo.GetNetworkName()) { + return true } } - return true + return false } diff --git a/go-controller/pkg/config/config.go b/go-controller/pkg/config/config.go index c7df666cbc..7cd97479c4 100644 --- a/go-controller/pkg/config/config.go +++ b/go-controller/pkg/config/config.go @@ -38,6 +38,9 @@ const DefaultVXLANPort = 4789 const DefaultDBTxnTimeout = time.Second * 100 +// DefaultEphemeralPortRange is used for unit testing only +const DefaultEphemeralPortRange = "32768-60999" + // The following are global config parameters that other modules may access directly var ( // Build information. Populated at build-time. @@ -494,6 +497,10 @@ type GatewayConfig struct { DisableForwarding bool `gcfg:"disable-forwarding"` // AllowNoUplink (disabled by default) controls if the external gateway bridge without an uplink port is allowed in local gateway mode. AllowNoUplink bool `gcfg:"allow-no-uplink"` + // EphemeralPortRange is the range of ports used by egress SNAT operations in OVN. Specifically for NAT where + // the source IP of the NAT will be a shared Node IP address. If unset, the value will be determined by sysctl lookup + // for the kernel's ephemeral range: net.ipv4.ip_local_port_range. Format is "-". + EphemeralPortRange string `gfcg:"ephemeral-port-range"` } // OvnAuthConfig holds client authentication and location details for @@ -664,6 +671,9 @@ func PrepareTestConfig() error { Kubernetes.DisableRequestedChassis = false EnableMulticast = false Default.OVSDBTxnTimeout = 5 * time.Second + if Gateway.Mode != GatewayModeDisabled { + Gateway.EphemeralPortRange = DefaultEphemeralPortRange + } if err := completeConfig(); err != nil { return err @@ -1509,6 +1519,14 @@ var OVNGatewayFlags = []cli.Flag{ Usage: "Allow the external gateway bridge without an uplink port in local gateway mode", Destination: &cliConfig.Gateway.AllowNoUplink, }, + &cli.StringFlag{ + Name: "ephemeral-port-range", + Usage: "The port range in '-' format for OVN to use when SNAT'ing to a node IP. " + + "This range should not collide with the node port range being used in Kubernetes. If not provided, " + + "the default value will be derived from checking the sysctl value of net.ipv4.ip_local_port_range on the node.", + Destination: &cliConfig.Gateway.EphemeralPortRange, + Value: Gateway.EphemeralPortRange, + }, // Deprecated CLI options &cli.BoolFlag{ Name: "init-gateways", @@ -1917,6 +1935,19 @@ func buildGatewayConfig(ctx *cli.Context, cli, file *config) error { if !found { return fmt.Errorf("invalid gateway mode %q: expect one of %s", string(Gateway.Mode), strings.Join(validModes, ",")) } + + if len(Gateway.EphemeralPortRange) > 0 { + if !isValidEphemeralPortRange(Gateway.EphemeralPortRange) { + return fmt.Errorf("invalid ephemeral-port-range, should be in the format -") + } + } else { + // auto-detect ephermal range + portRange, err := getKernelEphemeralPortRange() + if err != nil { + return fmt.Errorf("unable to auto-detect ephemeral port range to use with OVN") + } + Gateway.EphemeralPortRange = portRange + } } // Options are only valid if Mode is not disabled @@ -1927,6 +1958,9 @@ func buildGatewayConfig(ctx *cli.Context, cli, file *config) error { if Gateway.NextHop != "" { return fmt.Errorf("gateway next-hop option %q not allowed when gateway is disabled", Gateway.NextHop) } + if len(Gateway.EphemeralPortRange) > 0 { + return fmt.Errorf("gateway ephemeral port range option not allowed when gateway is disabled") + } } if Gateway.Mode != GatewayModeShared && Gateway.VLANID != 0 { diff --git a/go-controller/pkg/config/utils.go b/go-controller/pkg/config/utils.go index 7ff8eff484..f0f0ff1a6b 100644 --- a/go-controller/pkg/config/utils.go +++ b/go-controller/pkg/config/utils.go @@ -3,7 +3,9 @@ package config import ( "fmt" "net" + "os" "reflect" + "regexp" "strconv" "strings" @@ -328,3 +330,49 @@ func AllocateV6MasqueradeIPs(masqueradeSubnetNetworkAddress net.IP, masqueradeIP } return nil } + +func isValidEphemeralPortRange(s string) bool { + // Regex to match "-" with no extra characters + re := regexp.MustCompile(`^(\d{1,5})-(\d{1,5})$`) + matches := re.FindStringSubmatch(s) + if matches == nil { + return false + } + + minPort, err1 := strconv.Atoi(matches[1]) + maxPort, err2 := strconv.Atoi(matches[2]) + if err1 != nil || err2 != nil { + return false + } + + // Port numbers must be in the 1-65535 range + if minPort < 1 || minPort > 65535 || maxPort < 0 || maxPort > 65535 { + return false + } + + return maxPort > minPort +} + +func getKernelEphemeralPortRange() (string, error) { + data, err := os.ReadFile("/proc/sys/net/ipv4/ip_local_port_range") + if err != nil { + return "", fmt.Errorf("failed to read port range: %w", err) + } + + parts := strings.Fields(string(data)) + if len(parts) != 2 { + return "", fmt.Errorf("unexpected format: %q", string(data)) + } + + minPort, err := strconv.Atoi(parts[0]) + if err != nil { + return "", fmt.Errorf("invalid min port: %w", err) + } + + maxPort, err := strconv.Atoi(parts[1]) + if err != nil { + return "", fmt.Errorf("invalid max port: %w", err) + } + + return fmt.Sprintf("%d-%d", minPort, maxPort), nil +} diff --git a/go-controller/pkg/controllermanager/controller_manager.go b/go-controller/pkg/controllermanager/controller_manager.go index 06d88c4ce4..6b5f1e9f89 100644 --- a/go-controller/pkg/controllermanager/controller_manager.go +++ b/go-controller/pkg/controllermanager/controller_manager.go @@ -23,6 +23,7 @@ import ( libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics/recorders" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/observability" @@ -438,7 +439,7 @@ func (cm *ControllerManager) Start(ctx context.Context) error { // with k=10, // for a cluster with 10 nodes, measurement of 1 in every 100 requests // for a cluster with 100 nodes, measurement of 1 in every 1000 requests - metrics.GetConfigDurationRecorder().Run(cm.nbClient, cm.kube, 10, time.Second*5, cm.stopChan) + recorders.GetConfigDurationRecorder().Run(cm.nbClient, cm.watchFactory, 10, time.Second*5, cm.stopChan) } cm.podRecorder.Run(cm.sbClient, cm.stopChan) diff --git a/go-controller/pkg/controllermanager/node_controller_manager_test.go b/go-controller/pkg/controllermanager/node_controller_manager_test.go index cf96448fe9..92d51a25d8 100644 --- a/go-controller/pkg/controllermanager/node_controller_manager_test.go +++ b/go-controller/pkg/controllermanager/node_controller_manager_test.go @@ -228,7 +228,7 @@ var _ = Describe("Healthcheck tests", func() { }, } nodeList := []*corev1.Node{node} - factoryMock.On("GetNode", nodeName).Return(nodeList[0], nil) + factoryMock.On("GetNodeForWindows", nodeName).Return(nodeList[0], nil) factoryMock.On("GetNodes").Return(nodeList, nil) factoryMock.On("UserDefinedNetworkInformer").Return(nil) factoryMock.On("ClusterUserDefinedNetworkInformer").Return(nil) diff --git a/go-controller/pkg/kube/annotator_test.go b/go-controller/pkg/kube/annotator_test.go index 0caa0956a0..4c66adb81c 100644 --- a/go-controller/pkg/kube/annotator_test.go +++ b/go-controller/pkg/kube/annotator_test.go @@ -79,7 +79,7 @@ var _ = Describe("Annotator", func() { err := nodeAnnot.Run() Expect(err).ToNot(HaveOccurred()) - node, err := kube.GetNode(nodeName) + node, err := kube.GetNodeForWindows(nodeName) Expect(err).ToNot(HaveOccurred()) // should contain initial annotations diff --git a/go-controller/pkg/kube/kube.go b/go-controller/pkg/kube/kube.go index 4171e398e2..7eccec3d7f 100644 --- a/go-controller/pkg/kube/kube.go +++ b/go-controller/pkg/kube/kube.go @@ -12,7 +12,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" @@ -62,12 +61,11 @@ type Interface interface { PatchNode(old, new *corev1.Node) error UpdateNodeStatus(node *corev1.Node) error UpdatePodStatus(pod *corev1.Pod) error - GetAnnotationsOnPod(namespace, name string) (map[string]string, error) - GetNodes() ([]*corev1.Node, error) - GetNamespaces(labelSelector metav1.LabelSelector) ([]*corev1.Namespace, error) - GetPods(namespace string, opts metav1.ListOptions) ([]*corev1.Pod, error) - GetPod(namespace, name string) (*corev1.Pod, error) - GetNode(name string) (*corev1.Node, error) + // GetPodsForDBChecker should only be used by legacy DB checker. Use watchFactory instead to get pods. + GetPodsForDBChecker(namespace string, opts metav1.ListOptions) ([]*corev1.Pod, error) + // GetNodeForWindows should only be used for windows hybrid overlay binary and never in linux code + GetNodeForWindows(name string) (*corev1.Node, error) + GetNodesForWindows() ([]*corev1.Node, error) Events() kv1core.EventInterface } @@ -201,7 +199,7 @@ func (k *Kube) SetAnnotationsOnService(namespace, name string, annotations map[s // SetTaintOnNode tries to add a new taint to the node. If the taint already exists, it doesn't do anything. func (k *Kube) SetTaintOnNode(nodeName string, taint *corev1.Taint) error { - node, err := k.GetNode(nodeName) + node, err := k.GetNodeForWindows(nodeName) if err != nil { klog.Errorf("Unable to retrieve node %s for tainting %s: %v", nodeName, taint.ToString(), err) return err @@ -234,7 +232,7 @@ func (k *Kube) SetTaintOnNode(nodeName string, taint *corev1.Taint) error { // RemoveTaintFromNode removes all the taints that have the same key and effect from the node. // If the taint doesn't exist, it doesn't do anything. func (k *Kube) RemoveTaintFromNode(nodeName string, taint *corev1.Taint) error { - node, err := k.GetNode(nodeName) + node, err := k.GetNodeForWindows(nodeName) if err != nil { klog.Errorf("Unable to retrieve node %s for tainting %s: %v", nodeName, taint.ToString(), err) return err @@ -324,32 +322,8 @@ func (k *Kube) UpdatePodStatus(pod *corev1.Pod) error { return err } -// GetAnnotationsOnPod obtains the pod annotations from kubernetes apiserver, given the name and namespace -func (k *Kube) GetAnnotationsOnPod(namespace, name string) (map[string]string, error) { - pod, err := k.KClient.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - return pod.ObjectMeta.Annotations, nil -} - -// GetNamespaces returns the list of all Namespace objects matching the labelSelector -func (k *Kube) GetNamespaces(labelSelector metav1.LabelSelector) ([]*corev1.Namespace, error) { - list := []*corev1.Namespace{} - err := pager.New(func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { - return k.KClient.CoreV1().Namespaces().List(ctx, opts) - }).EachListItem(context.TODO(), metav1.ListOptions{ - LabelSelector: labels.Set(labelSelector.MatchLabels).String(), - ResourceVersion: "0", - }, func(obj runtime.Object) error { - list = append(list, obj.(*corev1.Namespace)) - return nil - }) - return list, err -} - -// GetPods returns the list of all Pod objects in a namespace matching the options -func (k *Kube) GetPods(namespace string, opts metav1.ListOptions) ([]*corev1.Pod, error) { +// GetPodsForDBChecker returns the list of all Pod objects in a namespace matching the options. Only used by the legacy db checker. +func (k *Kube) GetPodsForDBChecker(namespace string, opts metav1.ListOptions) ([]*corev1.Pod, error) { list := []*corev1.Pod{} opts.ResourceVersion = "0" err := pager.New(func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { @@ -361,13 +335,8 @@ func (k *Kube) GetPods(namespace string, opts metav1.ListOptions) ([]*corev1.Pod return list, err } -// GetPod obtains the pod from kubernetes apiserver, given the name and namespace -func (k *Kube) GetPod(namespace, name string) (*corev1.Pod, error) { - return k.KClient.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{}) -} - -// GetNodes returns the list of all Node objects from kubernetes -func (k *Kube) GetNodes() ([]*corev1.Node, error) { +// GetNodesForWindows returns the list of all Node objects from kubernetes. Only used by windows binary. +func (k *Kube) GetNodesForWindows() ([]*corev1.Node, error) { list := []*corev1.Node{} err := pager.New(func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { return k.KClient.CoreV1().Nodes().List(ctx, opts) @@ -380,8 +349,8 @@ func (k *Kube) GetNodes() ([]*corev1.Node, error) { return list, err } -// GetNode returns the Node resource from kubernetes apiserver, given its name -func (k *Kube) GetNode(name string) (*corev1.Node, error) { +// GetNodeForWindows returns the Node resource from kubernetes apiserver, given its name. Only used by windows binary. +func (k *Kube) GetNodeForWindows(name string) (*corev1.Node, error) { return k.KClient.CoreV1().Nodes().Get(context.TODO(), name, metav1.GetOptions{}) } diff --git a/go-controller/pkg/kube/kube_test.go b/go-controller/pkg/kube/kube_test.go index 4741cef0ed..d93119ff3a 100644 --- a/go-controller/pkg/kube/kube_test.go +++ b/go-controller/pkg/kube/kube_test.go @@ -96,7 +96,7 @@ var _ = Describe("Kube", func() { err := kube.SetTaintOnNode(node.Name, &taint) Expect(err).ToNot(HaveOccurred()) - updatedNode, err := kube.GetNode(node.Name) + updatedNode, err := kube.GetNodeForWindows(node.Name) Expect(err).ToNot(HaveOccurred()) Expect(updatedNode.Spec.Taints).To(Equal([]corev1.Taint{taint})) }) diff --git a/go-controller/pkg/kube/mocks/Interface.go b/go-controller/pkg/kube/mocks/Interface.go index 6631ca44c0..594d33d699 100644 --- a/go-controller/pkg/kube/mocks/Interface.go +++ b/go-controller/pkg/kube/mocks/Interface.go @@ -37,72 +37,12 @@ func (_m *Interface) Events() v1.EventInterface { return r0 } -// GetAnnotationsOnPod provides a mock function with given fields: namespace, name -func (_m *Interface) GetAnnotationsOnPod(namespace string, name string) (map[string]string, error) { - ret := _m.Called(namespace, name) - - if len(ret) == 0 { - panic("no return value specified for GetAnnotationsOnPod") - } - - var r0 map[string]string - var r1 error - if rf, ok := ret.Get(0).(func(string, string) (map[string]string, error)); ok { - return rf(namespace, name) - } - if rf, ok := ret.Get(0).(func(string, string) map[string]string); ok { - r0 = rf(namespace, name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(namespace, name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetNamespaces provides a mock function with given fields: labelSelector -func (_m *Interface) GetNamespaces(labelSelector metav1.LabelSelector) ([]*corev1.Namespace, error) { - ret := _m.Called(labelSelector) - - if len(ret) == 0 { - panic("no return value specified for GetNamespaces") - } - - var r0 []*corev1.Namespace - var r1 error - if rf, ok := ret.Get(0).(func(metav1.LabelSelector) ([]*corev1.Namespace, error)); ok { - return rf(labelSelector) - } - if rf, ok := ret.Get(0).(func(metav1.LabelSelector) []*corev1.Namespace); ok { - r0 = rf(labelSelector) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*corev1.Namespace) - } - } - - if rf, ok := ret.Get(1).(func(metav1.LabelSelector) error); ok { - r1 = rf(labelSelector) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetNode provides a mock function with given fields: name -func (_m *Interface) GetNode(name string) (*corev1.Node, error) { +func (_m *Interface) GetNodeForWindows(name string) (*corev1.Node, error) { ret := _m.Called(name) if len(ret) == 0 { - panic("no return value specified for GetNode") + panic("no return value specified for GetNodeForWindows") } var r0 *corev1.Node @@ -127,12 +67,12 @@ func (_m *Interface) GetNode(name string) (*corev1.Node, error) { return r0, r1 } -// GetNodes provides a mock function with given fields: -func (_m *Interface) GetNodes() ([]*corev1.Node, error) { +// GetNodesForWindows provides a mock function with given fields: +func (_m *Interface) GetNodesForWindows() ([]*corev1.Node, error) { ret := _m.Called() if len(ret) == 0 { - panic("no return value specified for GetNodes") + panic("no return value specified for GetNodesForWindows") } var r0 []*corev1.Node @@ -157,42 +97,12 @@ func (_m *Interface) GetNodes() ([]*corev1.Node, error) { return r0, r1 } -// GetPod provides a mock function with given fields: namespace, name -func (_m *Interface) GetPod(namespace string, name string) (*corev1.Pod, error) { - ret := _m.Called(namespace, name) - - if len(ret) == 0 { - panic("no return value specified for GetPod") - } - - var r0 *corev1.Pod - var r1 error - if rf, ok := ret.Get(0).(func(string, string) (*corev1.Pod, error)); ok { - return rf(namespace, name) - } - if rf, ok := ret.Get(0).(func(string, string) *corev1.Pod); ok { - r0 = rf(namespace, name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*corev1.Pod) - } - } - - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(namespace, name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetPods provides a mock function with given fields: namespace, opts -func (_m *Interface) GetPods(namespace string, opts metav1.ListOptions) ([]*corev1.Pod, error) { +func (_m *Interface) GetPodsForDBChecker(namespace string, opts metav1.ListOptions) ([]*corev1.Pod, error) { ret := _m.Called(namespace, opts) if len(ret) == 0 { - panic("no return value specified for GetPods") + panic("no return value specified for GetPodsForDBChecker") } var r0 []*corev1.Pod diff --git a/go-controller/pkg/kube/mocks/InterfaceOVN.go b/go-controller/pkg/kube/mocks/InterfaceOVN.go index 14a8a33af6..18e93ed800 100644 --- a/go-controller/pkg/kube/mocks/InterfaceOVN.go +++ b/go-controller/pkg/kube/mocks/InterfaceOVN.go @@ -91,36 +91,6 @@ func (_m *InterfaceOVN) Events() corev1.EventInterface { return r0 } -// GetAnnotationsOnPod provides a mock function with given fields: namespace, name -func (_m *InterfaceOVN) GetAnnotationsOnPod(namespace string, name string) (map[string]string, error) { - ret := _m.Called(namespace, name) - - if len(ret) == 0 { - panic("no return value specified for GetAnnotationsOnPod") - } - - var r0 map[string]string - var r1 error - if rf, ok := ret.Get(0).(func(string, string) (map[string]string, error)); ok { - return rf(namespace, name) - } - if rf, ok := ret.Get(0).(func(string, string) map[string]string); ok { - r0 = rf(namespace, name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(namespace, name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetEgressFirewalls provides a mock function with given fields: func (_m *InterfaceOVN) GetEgressFirewalls() ([]*egressfirewallv1.EgressFirewall, error) { ret := _m.Called() @@ -211,42 +181,12 @@ func (_m *InterfaceOVN) GetEgressIPs() ([]*egressipv1.EgressIP, error) { return r0, r1 } -// GetNamespaces provides a mock function with given fields: labelSelector -func (_m *InterfaceOVN) GetNamespaces(labelSelector metav1.LabelSelector) ([]*apicorev1.Namespace, error) { - ret := _m.Called(labelSelector) - - if len(ret) == 0 { - panic("no return value specified for GetNamespaces") - } - - var r0 []*apicorev1.Namespace - var r1 error - if rf, ok := ret.Get(0).(func(metav1.LabelSelector) ([]*apicorev1.Namespace, error)); ok { - return rf(labelSelector) - } - if rf, ok := ret.Get(0).(func(metav1.LabelSelector) []*apicorev1.Namespace); ok { - r0 = rf(labelSelector) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*apicorev1.Namespace) - } - } - - if rf, ok := ret.Get(1).(func(metav1.LabelSelector) error); ok { - r1 = rf(labelSelector) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetNode provides a mock function with given fields: name -func (_m *InterfaceOVN) GetNode(name string) (*apicorev1.Node, error) { +func (_m *InterfaceOVN) GetNodeForWindows(name string) (*apicorev1.Node, error) { ret := _m.Called(name) if len(ret) == 0 { - panic("no return value specified for GetNode") + panic("no return value specified for GetNodeForWindows") } var r0 *apicorev1.Node @@ -271,8 +211,8 @@ func (_m *InterfaceOVN) GetNode(name string) (*apicorev1.Node, error) { return r0, r1 } -// GetNodes provides a mock function with given fields: -func (_m *InterfaceOVN) GetNodes() ([]*apicorev1.Node, error) { +// GetNodesForWindows provides a mock function with given fields: +func (_m *InterfaceOVN) GetNodesForWindows() ([]*apicorev1.Node, error) { ret := _m.Called() if len(ret) == 0 { @@ -301,42 +241,12 @@ func (_m *InterfaceOVN) GetNodes() ([]*apicorev1.Node, error) { return r0, r1 } -// GetPod provides a mock function with given fields: namespace, name -func (_m *InterfaceOVN) GetPod(namespace string, name string) (*apicorev1.Pod, error) { - ret := _m.Called(namespace, name) - - if len(ret) == 0 { - panic("no return value specified for GetPod") - } - - var r0 *apicorev1.Pod - var r1 error - if rf, ok := ret.Get(0).(func(string, string) (*apicorev1.Pod, error)); ok { - return rf(namespace, name) - } - if rf, ok := ret.Get(0).(func(string, string) *apicorev1.Pod); ok { - r0 = rf(namespace, name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*apicorev1.Pod) - } - } - - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(namespace, name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetPods provides a mock function with given fields: namespace, opts -func (_m *InterfaceOVN) GetPods(namespace string, opts metav1.ListOptions) ([]*apicorev1.Pod, error) { +func (_m *InterfaceOVN) GetPodsForDBChecker(namespace string, opts metav1.ListOptions) ([]*apicorev1.Pod, error) { ret := _m.Called(namespace, opts) if len(ret) == 0 { - panic("no return value specified for GetPods") + panic("no return value specified for GetPodsForDBChecker") } var r0 []*apicorev1.Pod diff --git a/go-controller/pkg/libovsdb/ops/router.go b/go-controller/pkg/libovsdb/ops/router.go index da518f7cb3..3d5a6fc255 100644 --- a/go-controller/pkg/libovsdb/ops/router.go +++ b/go-controller/pkg/libovsdb/ops/router.go @@ -961,6 +961,10 @@ func buildNAT( Match: match, } + if config.Gateway.Mode != config.GatewayModeDisabled { + nat.ExternalPortRange = config.Gateway.EphemeralPortRange + } + if logicalPort != "" { nat.LogicalPort = &logicalPort } @@ -1061,7 +1065,7 @@ func isEquivalentNAT(existing *nbdb.NAT, searched *nbdb.NAT) bool { return false } - // Compre externalIP if its not empty. + // Compare externalIP if it's not empty. if searched.ExternalIP != "" && searched.ExternalIP != existing.ExternalIP { return false } diff --git a/go-controller/pkg/metrics/cluster_manager.go b/go-controller/pkg/metrics/cluster_manager.go index 3acba72759..f97a338b89 100644 --- a/go-controller/pkg/metrics/cluster_manager.go +++ b/go-controller/pkg/metrics/cluster_manager.go @@ -7,28 +7,29 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" ) var registerClusterManagerBaseMetrics sync.Once // MetricClusterManagerLeader identifies whether this instance of ovnkube-cluster-manager is a leader or not var MetricClusterManagerLeader = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemClusterManager, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, Name: "leader", Help: "Identifies whether the instance of ovnkube-cluster-manager is a leader(1) or not(0).", }) var MetricClusterManagerReadyDuration = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemClusterManager, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, Name: "ready_duration_seconds", Help: "The duration for the cluster manager to get to ready state", }) var metricV4HostSubnetCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemClusterManager, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, Name: "num_v4_host_subnets", Help: "The total number of v4 host subnets possible per network"}, []string{ @@ -37,8 +38,8 @@ var metricV4HostSubnetCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricV6HostSubnetCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemClusterManager, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, Name: "num_v6_host_subnets", Help: "The total number of v6 host subnets possible per network"}, []string{ @@ -47,8 +48,8 @@ var metricV6HostSubnetCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricV4AllocatedHostSubnetCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemClusterManager, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, Name: "allocated_v4_host_subnets", Help: "The total number of v4 host subnets currently allocated per network"}, []string{ @@ -57,8 +58,8 @@ var metricV4AllocatedHostSubnetCount = prometheus.NewGaugeVec(prometheus.GaugeOp ) var metricV6AllocatedHostSubnetCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemClusterManager, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, Name: "allocated_v6_host_subnets", Help: "The total number of v6 host subnets currently allocated per network"}, []string{ @@ -68,22 +69,22 @@ var metricV6AllocatedHostSubnetCount = prometheus.NewGaugeVec(prometheus.GaugeOp /** EgressIP metrics recorded from cluster-manager begins**/ var metricEgressIPCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemClusterManager, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, Name: "num_egress_ips", Help: "The number of defined egress IP addresses", }) var metricEgressIPNodeUnreacheableCount = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemClusterManager, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, Name: "egress_ips_node_unreachable_total", Help: "The total number of times assigned egress IP(s) were unreachable"}, ) var metricEgressIPRebalanceCount = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemClusterManager, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, Name: "egress_ips_rebalance_total", Help: "The total number of times assigned egress IP(s) needed to be moved to a different node"}, ) @@ -98,8 +99,8 @@ func RegisterClusterManagerBase() { prometheus.MustRegister(MetricClusterManagerReadyDuration) prometheus.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemClusterManager, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemClusterManager, Name: "build_info", Help: "A metric with a constant '1' value labeled by version, revision, branch, " + "and go version from which ovnkube was built and when and who built it", diff --git a/go-controller/pkg/metrics/metrics.go b/go-controller/pkg/metrics/metrics.go index 847619b8dd..ea86a65c0d 100644 --- a/go-controller/pkg/metrics/metrics.go +++ b/go-controller/pkg/metrics/metrics.go @@ -27,22 +27,11 @@ import ( libovsdbclient "github.com/ovn-org/libovsdb/client" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) const ( - MetricOvnkubeNamespace = "ovnkube" - MetricOvnkubeSubsystemController = "controller" - MetricOvnkubeSubsystemClusterManager = "clustermanager" - MetricOvnkubeSubsystemNode = "node" - MetricOvnNamespace = "ovn" - MetricOvnSubsystemDB = "db" - MetricOvnSubsystemNorthd = "northd" - MetricOvnSubsystemController = "controller" - MetricOvsNamespace = "ovs" - MetricOvsSubsystemVswitchd = "vswitchd" - MetricOvsSubsystemDB = "db" - ovnNorthd = "ovn-northd" ovnController = "ovn-controller" ovsVswitchd = "ovs-vswitchd" @@ -82,7 +71,7 @@ type stopwatchStatistics struct { // resource reached the maximum retry limit and will not be retried. This metric doesn't // need Subsystem string since it is applicable for both master and node. var MetricResourceRetryFailuresCount = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: MetricOvnkubeNamespace, + Namespace: types.MetricOvnkubeNamespace, Name: "resource_retry_failures_total", Help: "The total number of times processing a Kubernetes resource reached the maximum retry limit and was no longer processed", }) diff --git a/go-controller/pkg/metrics/node.go b/go-controller/pkg/metrics/node.go index 3b19c334d7..07e621fc97 100644 --- a/go-controller/pkg/metrics/node.go +++ b/go-controller/pkg/metrics/node.go @@ -7,13 +7,14 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" ) // MetricCNIRequestDuration is a prometheus metric that tracks the duration // of CNI requests var MetricCNIRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemNode, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemNode, Name: "cni_request_duration_seconds", Help: "The duration of CNI server requests.", Buckets: prometheus.ExponentialBuckets(.1, 2, 15)}, @@ -22,23 +23,23 @@ var MetricCNIRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOp ) var MetricNodeReadyDuration = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemNode, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemNode, Name: "ready_duration_seconds", Help: "The duration for the node to get to ready state.", }) var metricOvnNodePortEnabled = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemNode, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemNode, Name: "nodeport_enabled", Help: "Specifies if the node port is enabled on this node(1) or not(0).", }) // metric to get the size of ovnkube.log file var metricOvnKubeNodeLogFileSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemNode, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemNode, Name: "logfile_size_bytes", Help: "The size of ovnkube logfile on the node."}, []string{ @@ -56,8 +57,8 @@ func RegisterNodeMetrics(stopChan <-chan struct{}) { prometheus.MustRegister(metricOvnNodePortEnabled) prometheus.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemNode, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemNode, Name: "build_info", Help: "A metric with a constant '1' value labeled by version, revision, branch, " + "and go version from which ovnkube was built and when and who built it.", @@ -72,7 +73,7 @@ func RegisterNodeMetrics(stopChan <-chan struct{}) { }, func() float64 { return 1 }, )) - registerWorkqueueMetrics(MetricOvnkubeNamespace, MetricOvnkubeSubsystemNode) + registerWorkqueueMetrics(types.MetricOvnkubeNamespace, types.MetricOvnkubeSubsystemNode) if err := prometheus.Register(MetricResourceRetryFailuresCount); err != nil { if _, ok := err.(prometheus.AlreadyRegisteredError); !ok { panic(err) diff --git a/go-controller/pkg/metrics/ovn.go b/go-controller/pkg/metrics/ovn.go index 4cce457ea5..63f057e38f 100644 --- a/go-controller/pkg/metrics/ovn.go +++ b/go-controller/pkg/metrics/ovn.go @@ -12,29 +12,30 @@ import ( libovsdbclient "github.com/ovn-org/libovsdb/client" ovsops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops/ovs" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/vswitchd" ) // ovnController Configuration metrics var metricRemoteProbeInterval = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "remote_probe_interval_seconds", Help: "The inactivity probe interval of the connection to the OVN SB DB.", }) var metricOpenFlowProbeInterval = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "openflow_probe_interval_seconds", Help: "The inactivity probe interval of the OpenFlow connection to the " + "OpenvSwitch integration bridge.", }) var metricMonitorAll = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "monitor_all", Help: "Specifies if ovn-controller should monitor all records of tables in OVN SB DB. " + "If set to false, it will conditionally monitor the records that " + @@ -42,8 +43,8 @@ var metricMonitorAll = prometheus.NewGauge(prometheus.GaugeOpts{ }) var metricEncapIP = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "encap_ip", Help: "A metric with a constant '1' value labeled by ipadress that " + "specifies the encapsulation ip address configured on that node.", @@ -54,8 +55,8 @@ var metricEncapIP = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricSbConnectionMethod = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "sb_connection_method", Help: "A metric with a constant '1' value labeled by connection_method that " + "specifies the ovn-remote value configured on that node.", @@ -66,8 +67,8 @@ var metricSbConnectionMethod = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricEncapType = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "encap_type", Help: "A metric with a constant '1' value labeled by type that " + "specifies the encapsulation type a chassis should use to " + @@ -79,8 +80,8 @@ var metricEncapType = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricBridgeMappings = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "bridge_mappings", Help: "A metric with a constant '1' value labeled by mapping that " + "specifies list of key-value pairs that map a physical network name " + @@ -92,8 +93,8 @@ var metricBridgeMappings = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOVNControllerSBDBConnection = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "southbound_database_connected", Help: "Specifies if OVN controller is connected to OVN southbound database (1) or not (0)", }) @@ -261,11 +262,11 @@ func setOvnControllerConfigurationMetrics(ovsDBClient libovsdbclient.Client) (er } openflowProbeField := openvSwitch.ExternalIDs["ovn-bridge-remote-probe-interval"] - openflowProbeVal := parseMetricToFloat(MetricOvnSubsystemController, "ovn-bridge-remote-probe-interval", openflowProbeField) + openflowProbeVal := parseMetricToFloat(types.MetricOvnSubsystemController, "ovn-bridge-remote-probe-interval", openflowProbeField) metricOpenFlowProbeInterval.Set(openflowProbeVal) remoteProbeField := openvSwitch.ExternalIDs["ovn-remote-probe-interval"] - remoteProbeValue := parseMetricToFloat(MetricOvnSubsystemController, "ovn-remote-probe-interval", remoteProbeField) + remoteProbeValue := parseMetricToFloat(types.MetricOvnSubsystemController, "ovn-remote-probe-interval", remoteProbeField) metricRemoteProbeInterval.Set(remoteProbeValue / 1000) var ovnMonitorValue float64 @@ -406,8 +407,8 @@ func RegisterOvnControllerMetrics(ovsDBClient libovsdbclient.Client, getOvnControllerVersionInfo() ovnRegistry.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "build_info", Help: "A metric with a constant '1' value labeled by version and library " + "from which ovn binaries were built", @@ -423,8 +424,8 @@ func RegisterOvnControllerMetrics(ovsDBClient libovsdbclient.Client, ovnRegistry.MustRegister(metricOVNControllerSBDBConnection) ovnRegistry.MustRegister(prometheus.NewCounterFunc( prometheus.CounterOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "integration_bridge_openflow_total", Help: "The total number of OpenFlow flows in the integration bridge.", }, func() float64 { @@ -437,7 +438,7 @@ func RegisterOvnControllerMetrics(ovsDBClient libovsdbclient.Client, for _, kvPair := range strings.Fields(stdout) { if strings.HasPrefix(kvPair, "flow_count=") { value := strings.Split(kvPair, "=")[1] - return parseMetricToFloat(MetricOvnSubsystemController, "integration_bridge_openflow_total", + return parseMetricToFloat(types.MetricOvnSubsystemController, "integration_bridge_openflow_total", value) } } @@ -445,8 +446,8 @@ func RegisterOvnControllerMetrics(ovsDBClient libovsdbclient.Client, })) ovnRegistry.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "integration_bridge_patch_ports", Help: "Captures the number of patch ports that connect br-int OVS " + "bridge to physical OVS bridge and br-local OVS bridge.", @@ -456,8 +457,8 @@ func RegisterOvnControllerMetrics(ovsDBClient libovsdbclient.Client, })) ovnRegistry.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemController, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemController, Name: "integration_bridge_geneve_ports", Help: "Captures the number of geneve ports that are on br-int OVS bridge.", }, @@ -475,11 +476,11 @@ func RegisterOvnControllerMetrics(ovsDBClient libovsdbclient.Client, ovnRegistry.MustRegister(metricBridgeMappings) // Register the ovn-controller coverage/show metrics componentCoverageShowMetricsMap[ovnController] = ovnControllerCoverageShowMetricsMap - registerCoverageShowMetrics(ovnController, MetricOvnNamespace, MetricOvnSubsystemController) + registerCoverageShowMetrics(ovnController, types.MetricOvnNamespace, types.MetricOvnSubsystemController) // Register the ovn-controller coverage/show metrics componentStopwatchShowMetricsMap[ovnController] = ovnControllerStopwatchShowMetricsMap - registerStopwatchShowMetrics(ovnController, MetricOvnNamespace, MetricOvnSubsystemController) + registerStopwatchShowMetrics(ovnController, types.MetricOvnNamespace, types.MetricOvnSubsystemController) // ovn-controller configuration metrics updater go ovnControllerConfigurationMetricsUpdater(ovsDBClient, diff --git a/go-controller/pkg/metrics/ovn_db.go b/go-controller/pkg/metrics/ovn_db.go index 8116a68732..ea206adc3c 100644 --- a/go-controller/pkg/metrics/ovn_db.go +++ b/go-controller/pkg/metrics/ovn_db.go @@ -14,12 +14,13 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) var metricOVNDBSessions = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "jsonrpc_server_sessions", Help: "Active number of JSON RPC Server sessions to the DB"}, []string{ @@ -28,8 +29,8 @@ var metricOVNDBSessions = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOVNDBMonitor = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "ovsdb_monitors", Help: "Number of OVSDB Monitors on the server"}, []string{ @@ -38,8 +39,8 @@ var metricOVNDBMonitor = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "db_size_bytes", Help: "The size of the database file associated with the OVN DB component."}, []string{ @@ -49,8 +50,8 @@ var metricDBSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ // ClusterStatus metrics var metricDBClusterCID = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_id", Help: "A metric with a constant '1' value labeled by database name and cluster uuid"}, []string{ @@ -60,8 +61,8 @@ var metricDBClusterCID = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterSID = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_server_id", Help: "A metric with a constant '1' value labeled by database name, cluster uuid " + "and server uuid"}, @@ -73,8 +74,8 @@ var metricDBClusterSID = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterServerStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_server_status", Help: "A metric with a constant '1' value labeled by database name, cluster uuid, server uuid " + "server status"}, @@ -87,8 +88,8 @@ var metricDBClusterServerStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterServerRole = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_server_role", Help: "A metric with a constant '1' value labeled by database name, cluster uuid, server uuid " + "and server role"}, @@ -101,8 +102,8 @@ var metricDBClusterServerRole = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterTerm = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_term", Help: "A metric that returns the current election term value labeled by database name, cluster uuid, and " + "server uuid"}, @@ -114,8 +115,8 @@ var metricDBClusterTerm = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterServerVote = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_server_vote", Help: "A metric with a constant '1' value labeled by database name, cluster uuid, server uuid " + "and server vote"}, @@ -128,8 +129,8 @@ var metricDBClusterServerVote = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterElectionTimer = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_election_timer", Help: "A metric that returns the current election timer value labeled by database name, cluster uuid, " + "and server uuid"}, @@ -141,8 +142,8 @@ var metricDBClusterElectionTimer = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterLogIndexStart = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_log_index_start", Help: "A metric that returns the log entry index start value labeled by database name, cluster uuid, " + "and server uuid"}, @@ -154,8 +155,8 @@ var metricDBClusterLogIndexStart = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterLogIndexNext = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_log_index_next", Help: "A metric that returns the log entry index next value labeled by database name, cluster uuid, " + "and server uuid"}, @@ -167,8 +168,8 @@ var metricDBClusterLogIndexNext = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterLogNotCommitted = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_log_not_committed", Help: "A metric that returns the number of log entries not committed labeled by database name, cluster uuid, " + "and server uuid"}, @@ -180,8 +181,8 @@ var metricDBClusterLogNotCommitted = prometheus.NewGaugeVec(prometheus.GaugeOpts ) var metricDBClusterLogNotApplied = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_log_not_applied", Help: "A metric that returns the number of log entries not applied labeled by database name, cluster uuid, " + "and server uuid"}, @@ -193,8 +194,8 @@ var metricDBClusterLogNotApplied = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterConnIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_inbound_connections_total", Help: "A metric that returns the total number of inbound connections to the server labeled by " + "database name, cluster uuid, and server uuid"}, @@ -206,8 +207,8 @@ var metricDBClusterConnIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterConnOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_outbound_connections_total", Help: "A metric that returns the total number of outbound connections from the server labeled by " + "database name, cluster uuid, and server uuid"}, @@ -219,8 +220,8 @@ var metricDBClusterConnOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterConnInErr = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_inbound_connections_error_total", Help: "A metric that returns the total number of failed inbound connections to the server labeled by " + " database name, cluster uuid, and server uuid"}, @@ -232,8 +233,8 @@ var metricDBClusterConnInErr = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricDBClusterConnOutErr = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "cluster_outbound_connections_error_total", Help: "A metric that returns the total number of failed outbound connections from the server labeled by " + "database name, cluster uuid, and server uuid"}, @@ -382,8 +383,8 @@ func RegisterOvnDBMetrics(clientset kubernetes.Interface, k8sNodeName string, st ovnRegistry.MustRegister(metricOVNDBSessions) ovnRegistry.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "build_info", Help: "A metric with a constant '1' value labeled by ovsdb-server version and " + "NB and SB schema version", diff --git a/go-controller/pkg/metrics/ovn_northd.go b/go-controller/pkg/metrics/ovn_northd.go index ae2afe45c8..e72c89fdd3 100644 --- a/go-controller/pkg/metrics/ovn_northd.go +++ b/go-controller/pkg/metrics/ovn_northd.go @@ -11,6 +11,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -128,8 +129,8 @@ func RegisterOvnNorthdMetrics(clientset kubernetes.Interface, k8sNodeName string getOvnNorthdVersionInfo() ovnRegistry.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemNorthd, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemNorthd, Name: "build_info", Help: "A metric with a constant '1' value labeled by version and library " + "from which ovn binaries were built", @@ -142,8 +143,8 @@ func RegisterOvnNorthdMetrics(clientset kubernetes.Interface, k8sNodeName string )) ovnRegistry.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemNorthd, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemNorthd, Name: "status", Help: "Specifies whether this instance of ovn-northd is standby(0) or active(1) or paused(2).", }, func() float64 { @@ -169,8 +170,8 @@ func RegisterOvnNorthdMetrics(clientset kubernetes.Interface, k8sNodeName string )) ovnRegistry.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemNorthd, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemNorthd, Name: "nb_connection_status", Help: "Specifies nb-connection-status of ovn-northd, not connected(0) or connected(1).", }, func() float64 { @@ -179,8 +180,8 @@ func RegisterOvnNorthdMetrics(clientset kubernetes.Interface, k8sNodeName string )) ovnRegistry.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemNorthd, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemNorthd, Name: "sb_connection_status", Help: "Specifies sb-connection-status of ovn-northd, not connected(0) or connected(1).", }, func() float64 { @@ -190,11 +191,11 @@ func RegisterOvnNorthdMetrics(clientset kubernetes.Interface, k8sNodeName string // Register the ovn-northd coverage/show metrics with prometheus componentCoverageShowMetricsMap[ovnNorthd] = ovnNorthdCoverageShowMetricsMap - registerCoverageShowMetrics(ovnNorthd, MetricOvnNamespace, MetricOvnSubsystemNorthd) + registerCoverageShowMetrics(ovnNorthd, types.MetricOvnNamespace, types.MetricOvnSubsystemNorthd) go coverageShowMetricsUpdater(ovnNorthd, stopChan) // Register the ovn-northd stopwatch/show metrics with prometheus componentStopwatchShowMetricsMap[ovnNorthd] = ovnNorthdStopwatchShowMetricsMap - registerStopwatchShowMetrics(ovnNorthd, MetricOvnNamespace, MetricOvnSubsystemNorthd) + registerStopwatchShowMetrics(ovnNorthd, types.MetricOvnNamespace, types.MetricOvnSubsystemNorthd) go stopwatchShowMetricsUpdater(ovnNorthd, stopChan) } diff --git a/go-controller/pkg/metrics/ovnkube_controller.go b/go-controller/pkg/metrics/ovnkube_controller.go index b73a0ad8de..a4cb9fd693 100644 --- a/go-controller/pkg/metrics/ovnkube_controller.go +++ b/go-controller/pkg/metrics/ovnkube_controller.go @@ -3,8 +3,6 @@ package metrics import ( "errors" "fmt" - "hash/fnv" - "math" "runtime" "strconv" "sync" @@ -21,14 +19,13 @@ import ( "github.com/ovn-org/libovsdb/cache" libovsdbclient "github.com/ovn-org/libovsdb/client" "github.com/ovn-org/libovsdb/model" - "github.com/ovn-org/libovsdb/ovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/sbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -37,16 +34,16 @@ import ( // read from SB DB. This is registered within func RunTimestamp in order to allow gathering this // metric on the fly when metrics are scraped. var metricNbE2eTimestamp = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "nb_e2e_timestamp", Help: "The current e2e-timestamp value as written to the northbound database"}, ) // metricDbTimestamp is the UNIX timestamp seen in NB and SB DBs. var metricDbTimestamp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemDB, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemDB, Name: "e2e_timestamp", Help: "The current e2e-timestamp value as observed in this instance of the database"}, []string{ @@ -57,8 +54,8 @@ var metricDbTimestamp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ // metricPodCreationLatency is the time between a pod being scheduled and // completing its logical switch port configuration. var metricPodCreationLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "pod_creation_latency_seconds", Help: "The duration between a pod being scheduled and completing its logical switch port configuration", Buckets: prometheus.ExponentialBuckets(.1, 2, 15), @@ -66,8 +63,8 @@ var metricPodCreationLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ // MetricResourceUpdateCount is the number of times a particular resource's UpdateFunc has been called. var MetricResourceUpdateCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "resource_update_total", Help: "The number of times a given resource event (add, update, or delete) has been handled"}, []string{ @@ -79,8 +76,8 @@ var MetricResourceUpdateCount = prometheus.NewCounterVec(prometheus.CounterOpts{ // MetricResourceAddLatency is the time taken to complete resource update by an handler. // This measures the latency for all of the handlers for a given resource. var MetricResourceAddLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "resource_add_latency_seconds", Help: "The duration to process all handlers for a given resource event - add.", Buckets: prometheus.ExponentialBuckets(.1, 2, 15)}, @@ -89,8 +86,8 @@ var MetricResourceAddLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ // MetricResourceUpdateLatency is the time taken to complete resource update by an handler. // This measures the latency for all of the handlers for a given resource. var MetricResourceUpdateLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "resource_update_latency_seconds", Help: "The duration to process all handlers for a given resource event - update.", Buckets: prometheus.ExponentialBuckets(.1, 2, 15)}, @@ -99,8 +96,8 @@ var MetricResourceUpdateLatency = prometheus.NewHistogram(prometheus.HistogramOp // MetricResourceDeleteLatency is the time taken to complete resource update by an handler. // This measures the latency for all of the handlers for a given resource. var MetricResourceDeleteLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "resource_delete_latency_seconds", Help: "The duration to process all handlers for a given resource event - delete.", Buckets: prometheus.ExponentialBuckets(.1, 2, 15)}, @@ -108,32 +105,32 @@ var MetricResourceDeleteLatency = prometheus.NewHistogram(prometheus.HistogramOp // MetricRequeueServiceCount is the number of times a particular service has been requeued. var MetricRequeueServiceCount = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "requeue_service_total", Help: "A metric that captures the number of times a service is requeued after failing to sync with OVN"}, ) // MetricSyncServiceCount is the number of times a particular service has been synced. var MetricSyncServiceCount = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "sync_service_total", Help: "A metric that captures the number of times a service is synced with OVN load balancers"}, ) // MetricSyncServiceLatency is the time taken to sync a service with the OVN load balancers. var MetricSyncServiceLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "sync_service_latency_seconds", Help: "The latency of syncing a service with the OVN load balancers", Buckets: prometheus.ExponentialBuckets(.1, 2, 15)}, ) var MetricOVNKubeControllerReadyDuration = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "ready_duration_seconds", Help: "The duration for the ovnkube-controller to get to ready state", }) @@ -141,8 +138,8 @@ var MetricOVNKubeControllerReadyDuration = prometheus.NewGauge(prometheus.GaugeO // MetricOVNKubeControllerSyncDuration is the time taken to complete initial Watch for different resource. // Resource name is in the label. var MetricOVNKubeControllerSyncDuration = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "sync_duration_seconds", Help: "The duration to sync and setup all handlers for a given resource"}, []string{ @@ -151,15 +148,15 @@ var MetricOVNKubeControllerSyncDuration = prometheus.NewGaugeVec(prometheus.Gaug // MetricOVNKubeControllerLeader identifies whether this instance of ovnkube-controller is a leader or not var MetricOVNKubeControllerLeader = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "leader", Help: "Identifies whether the instance of ovnkube-controller is a leader(1) or not(0).", }) var metricOvnKubeControllerLogFileSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "logfile_size_bytes", Help: "The size of ovnkube-controller log file."}, []string{ @@ -168,24 +165,24 @@ var metricOvnKubeControllerLogFileSize = prometheus.NewGaugeVec(prometheus.Gauge ) var metricEgressIPAssignLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "egress_ips_assign_latency_seconds", Help: "The latency of egress IP assignment to ovn nb database", Buckets: prometheus.ExponentialBuckets(.001, 2, 15), }) var metricEgressIPUnassignLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "egress_ips_unassign_latency_seconds", Help: "The latency of egress IP unassignment from ovn nb database", Buckets: prometheus.ExponentialBuckets(.001, 2, 15), }) var metricNetpolEventLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "network_policy_event_latency_seconds", Help: "The latency of full network policy event handling (create, delete)", Buckets: prometheus.ExponentialBuckets(.004, 2, 15)}, @@ -194,8 +191,8 @@ var metricNetpolEventLatency = prometheus.NewHistogramVec(prometheus.HistogramOp }) var metricNetpolLocalPodEventLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "network_policy_local_pod_event_latency_seconds", Help: "The latency of local pod events handling (add, delete)", Buckets: prometheus.ExponentialBuckets(.002, 2, 15)}, @@ -204,8 +201,8 @@ var metricNetpolLocalPodEventLatency = prometheus.NewHistogramVec(prometheus.His }) var metricNetpolPeerNamespaceEventLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "network_policy_peer_namespace_event_latency_seconds", Help: "The latency of peer namespace events handling (add, delete)", Buckets: prometheus.ExponentialBuckets(.002, 2, 15)}, @@ -214,8 +211,8 @@ var metricNetpolPeerNamespaceEventLatency = prometheus.NewHistogramVec(prometheu }) var metricPodSelectorAddrSetPodEventLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "pod_selector_address_set_pod_event_latency_seconds", Help: "The latency of peer pod events handling (add, delete)", Buckets: prometheus.ExponentialBuckets(.002, 2, 15)}, @@ -224,8 +221,8 @@ var metricPodSelectorAddrSetPodEventLatency = prometheus.NewHistogramVec(prometh }) var metricPodSelectorAddrSetNamespaceEventLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "pod_selector_address_set_namespace_event_latency_seconds", Help: "The latency of peer namespace events handling (add, delete)", Buckets: prometheus.ExponentialBuckets(.002, 2, 15)}, @@ -234,8 +231,8 @@ var metricPodSelectorAddrSetNamespaceEventLatency = prometheus.NewHistogramVec(p }) var metricPodEventLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "pod_event_latency_seconds", Help: "The latency of pod events handling (add, update, delete)", Buckets: prometheus.ExponentialBuckets(.002, 2, 15)}, @@ -244,51 +241,51 @@ var metricPodEventLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ }) var metricEgressFirewallRuleCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "num_egress_firewall_rules", Help: "The number of egress firewall rules defined"}, ) var metricIPsecEnabled = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "ipsec_enabled", Help: "Specifies whether IPSec is enabled for this cluster(1) or not enabled for this cluster(0)", }) var metricEgressRoutingViaHost = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "egress_routing_via_host", Help: "Specifies whether egress gateway mode is via host networking stack(1) or not(0)", }) var metricEgressFirewallCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "num_egress_firewalls", Help: "The number of egress firewall policies", }) /** AdminNetworkPolicyMetrics Begin**/ var metricANPCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "admin_network_policies", Help: "The total number of admin network policies in the cluster", }) var metricBANPCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "baseline_admin_network_policies", Help: "The total number of baseline admin network policies in the cluster", }) var metricANPDBObjects = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "admin_network_policies_db_objects", Help: "The total number of OVN NBDB objects (table_name) owned by AdminNetworkPolicy controller in the cluster"}, []string{ @@ -297,8 +294,8 @@ var metricANPDBObjects = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricBANPDBObjects = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "baseline_admin_network_policies_db_objects", Help: "The total number of OVN NBDB objects (table_name) owned by BaselineAdminNetworkPolicy controller in the cluster"}, []string{ @@ -310,64 +307,37 @@ var metricBANPDBObjects = prometheus.NewGaugeVec(prometheus.GaugeOpts{ // metricFirstSeenLSPLatency is the time between a pod first seen in OVN-Kubernetes and its Logical Switch Port is created var metricFirstSeenLSPLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "pod_first_seen_lsp_created_duration_seconds", Help: "The duration between a pod first observed in OVN-Kubernetes and Logical Switch Port created", Buckets: prometheus.ExponentialBuckets(.01, 2, 15), }) var metricLSPPortBindingLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "pod_lsp_created_port_binding_duration_seconds", Help: "The duration between a pods Logical Switch Port created and port binding observed in cache", Buckets: prometheus.ExponentialBuckets(.01, 2, 15), }) var metricPortBindingChassisLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "pod_port_binding_port_binding_chassis_duration_seconds", Help: "The duration between a pods port binding observed and port binding chassis update observed in cache", Buckets: prometheus.ExponentialBuckets(.01, 2, 15), }) var metricPortBindingUpLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "pod_port_binding_chassis_port_binding_up_duration_seconds", Help: "The duration between a pods port binding chassis update and port binding up observed in cache", Buckets: prometheus.ExponentialBuckets(.01, 2, 15), }) -var metricNetworkProgramming prometheus.ObserverVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, - Name: "network_programming_duration_seconds", - Help: "The duration to apply network configuration for a kind (e.g. pod, service, networkpolicy). " + - "Configuration includes add, update and delete events for each kind.", - Buckets: merge( - prometheus.LinearBuckets(0.25, 0.25, 2), // 0.25s, 0.50s - prometheus.LinearBuckets(1, 1, 59), // 1s, 2s, 3s, ... 59s - prometheus.LinearBuckets(60, 5, 12), // 60s, 65s, 70s, ... 115s - prometheus.LinearBuckets(120, 30, 11))}, // 2min, 2.5min, 3min, ..., 7min - []string{ - "kind", - }) - -var metricNetworkProgrammingOVN = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, - Name: "network_programming_ovn_duration_seconds", - Help: "The duration for OVN to apply network configuration", - Buckets: merge( - prometheus.LinearBuckets(0.25, 0.25, 2), // 0.25s, 0.50s - prometheus.LinearBuckets(1, 1, 59), // 1s, 2s, 3s, ... 59s - prometheus.LinearBuckets(60, 5, 12), // 60s, 65s, 70s, ... 115s - prometheus.LinearBuckets(120, 30, 11))}, // 2min, 2.5min, 3min, ..., 7min -) - const ( globalOptionsTimestampField = "e2e_timestamp" globalOptionsProbeIntervalField = "northd_probe_interval" @@ -381,8 +351,8 @@ func RegisterOVNKubeControllerBase() { prometheus.MustRegister(MetricOVNKubeControllerSyncDuration) prometheus.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "build_info", Help: "A metric with a constant '1' value labeled by version, revision, branch, " + "and go version from which ovnkube was built and when and who built it", @@ -410,11 +380,11 @@ func RegisterOVNKubeControllerPerformance(nbClient libovsdbclient.Client) { prometheus.MustRegister(MetricRequeueServiceCount) prometheus.MustRegister(MetricSyncServiceCount) prometheus.MustRegister(MetricSyncServiceLatency) - registerWorkqueueMetrics(MetricOvnkubeNamespace, MetricOvnkubeSubsystemController) + registerWorkqueueMetrics(types.MetricOvnkubeNamespace, types.MetricOvnkubeSubsystemController) prometheus.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnNamespace, - Subsystem: MetricOvnSubsystemNorthd, + Namespace: types.MetricOvnNamespace, + Subsystem: types.MetricOvnSubsystemNorthd, Name: "northd_probe_interval", Help: "The maximum number of milliseconds of idle time on connection to the OVN SB " + "and NB DB before sending an inactivity probe message", @@ -505,8 +475,8 @@ func RunTimestamp(stopChan <-chan struct{}, sbClient, nbClient libovsdbclient.Cl // cache when metrics HTTP endpoint is scraped. prometheus.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvnkubeNamespace, - Subsystem: MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "sb_e2e_timestamp", Help: "The current e2e-timestamp value as observed in the southbound database", }, func() float64 { @@ -916,521 +886,6 @@ func getPodUIDFromPortBinding(row *sbdb.PortBinding) kapimtypes.UID { return kapimtypes.UID(podUID) } -const ( - updateOVNMeasurementChSize = 500 - deleteOVNMeasurementChSize = 50 - processChSize = 1000 - nbGlobalTable = "NB_Global" - //fixme: remove when bug is fixed in OVN (Red Hat bugzilla bug number 2074019). Also, handle overflow event. - maxNbCfg = math.MaxUint32 - 1000 - maxMeasurementLifetime = 20 * time.Minute -) - -type ovnMeasurement struct { - // time just before ovsdb tx is called - startTimestamp time.Time - // time when the nbCfg value and its associated configuration is applied to all nodes - endTimestamp time.Time - // OVN measurement complete - start and end timestamps are valid - complete bool - // nb_cfg value that started the measurement - nbCfg int -} - -// measurement stores a measurement attempt through OVN-Kubernetes controller and optionally OVN -type measurement struct { - // kubernetes kind e.g. pod or service - kind string - // time when Add is executed - startTimestamp time.Time - // time when End is executed - endTimestamp time.Time - // if true endTimestamp is valid - end bool - // time when this measurement expires. Set during Add - expiresAt time.Time - // OVN measurement(s) via AddOVN - ovnMeasurements []ovnMeasurement -} - -// hvCfgUpdate holds the information received from OVN Northbound event handler -type hvCfgUpdate struct { - // timestamp is in milliseconds - timestamp int - hvCfg int -} - -type ConfigDurationRecorder struct { - // rate at which measurements are allowed. Probabilistically, 1 in every measurementRate - measurementRate uint64 - measurements map[string]measurement - // controls RW access to measurements map - measurementsMu sync.RWMutex - // channel to trigger processing a measurement following call to End func. Channel string is kind/namespace/name - triggerProcessCh chan string - enabled bool -} - -// global variable is needed because this functionality is accessed in many functions -var cdr *ConfigDurationRecorder - -// lock for accessing the cdr global variable -var cdrMutex sync.Mutex - -func GetConfigDurationRecorder() *ConfigDurationRecorder { - cdrMutex.Lock() - defer cdrMutex.Unlock() - if cdr == nil { - cdr = &ConfigDurationRecorder{} - } - return cdr -} - -var configDurationRegOnce sync.Once - -// Run monitors the config duration for OVN-Kube master to configure k8 kinds. A measurement maybe allowed and this is -// related to the number of k8 nodes, N [1] and by argument k [2] where there is a probability that 1 out of N*k -// measurement attempts are allowed. If k=0, all measurements are allowed. mUpdatePeriod determines the period to -// process and publish metrics -// [1] 1 0. The measurement rate is proportional to - // the number of nodes, N and argument k. 1 out of every N*k attempted measurements will succeed. - - // For the optional OVN measurement by calling AddOVN, when the CMS is about to make a transaction to configure - // whatever kind, a call to AddOVN function allows the caller to measure OVN duration. - // An ovsdb operation is returned to the caller of AddOVN, which they can bundle with their existing transactions - // sent to OVN which will tell OVN to measure how long it takes to configure all nodes with the config in the transaction. - // Config duration then waits for OVN to configure all nodes and calculates the time delta. - - // ** configuration duration recorder - caveats ** - // For the optional OVN recording, it does not give you an exact time duration for how long it takes to configure your - // k8 kind. When you are recording how long it takes OVN to complete your configuration to all nodes, other - // transactions may have occurred which may increases the overall time. You may also get longer processing times if one - // or more nodes are unavailable because we are measuring how long the functionality takes to apply to ALL nodes. - - // ** configuration duration recorder - How the duration of the config is measured within OVN ** - // We increment the nb_cfg integer value in the NB_Global table. - // ovn-northd notices the nb_cfg change and copies the nb_cfg value to SB_Global table field nb_cfg along with any - // other configuration that is changed in OVN Northbound database. - // All ovn-controllers detect nb_cfg value change and generate a 'barrier' on the openflow connection to the - // nodes ovs-vswitchd. Once ovn-controllers receive the 'barrier processed' reply from ovs-vswitchd which - // indicates that all relevant openflow operations associated with NB_Globals nb_cfg value have been - // propagated to the nodes OVS, it copies the SB_Global nb_cfg value to its Chassis_Private table nb_cfg record. - // ovn-northd detects changes to the Chassis_Private startRecords and computes the minimum nb_cfg for all Chassis_Private - // nb_cfg and stores this in NB_Global hv_cfg field along with a timestamp to field hv_cfg_timestamp which - // reflects the time when the slowest chassis catches up with the northbound configuration. - configDurationRegOnce.Do(func() { - prometheus.MustRegister(metricNetworkProgramming) - prometheus.MustRegister(metricNetworkProgrammingOVN) - }) - - cr.measurements = make(map[string]measurement) - // watch node count and adjust measurement rate if node count changes - cr.runMeasurementRateAdjuster(kube, k, time.Hour, stop) - // we currently do not clean the following channels up upon exit - cr.triggerProcessCh = make(chan string, processChSize) - updateOVNMeasurementCh := make(chan hvCfgUpdate, updateOVNMeasurementChSize) - deleteOVNMeasurementCh := make(chan int, deleteOVNMeasurementChSize) - go cr.processMeasurements(workerLoopPeriod, updateOVNMeasurementCh, deleteOVNMeasurementCh, stop) - - nbClient.Cache().AddEventHandler(&cache.EventHandlerFuncs{ - UpdateFunc: func(table string, old model.Model, new model.Model) { - if table != nbGlobalTable { - return - } - oldRow := old.(*nbdb.NBGlobal) - newRow := new.(*nbdb.NBGlobal) - - if oldRow.HvCfg != newRow.HvCfg && oldRow.HvCfgTimestamp != newRow.HvCfgTimestamp && newRow.HvCfgTimestamp > 0 { - select { - case updateOVNMeasurementCh <- hvCfgUpdate{hvCfg: newRow.HvCfg, timestamp: newRow.HvCfgTimestamp}: - default: - klog.Warning("Config duration recorder: unable to update OVN measurement") - select { - case deleteOVNMeasurementCh <- newRow.HvCfg: - default: - } - } - } - }, - }) - cr.enabled = true -} - -// Start allows the caller to attempt measurement of a control plane configuration duration, as a metric, -// the duration between functions Start and End. Optionally, if you wish to record OVN config duration, -// call AddOVN which will add the duration for OVN to apply the configuration to all nodes. -// The caller must pass kind,namespace,name which will be used to determine if the object -// is allowed to record. To allow no locking, each go routine that calls this function, can determine itself -// if it is allowed to measure. -// There is a mandatory two-step process to complete a measurement. -// Step 1) Call Start when you wish to begin a measurement - ideally when processing for the object starts -// Step 2) Call End which will complete a measurement -// Optionally, call AddOVN when you are making a transaction to OVN in order to add on the OVN duration to an existing -// measurement. This must be called between Start and End. Not every call to Start will result in a measurement -// and the rate of measurements depends on the number of nodes and function Run arg k. -// Only one measurement for a kind/namespace/name is allowed until the current measurement is Ended (via End) and -// processed. This is guaranteed by workqueues (even with multiple workers) and informer event handlers. -func (cr *ConfigDurationRecorder) Start(kind, namespace, name string) (time.Time, bool) { - if !cr.enabled { - return time.Time{}, false - } - kindNamespaceName := fmt.Sprintf("%s/%s/%s", kind, namespace, name) - if !cr.allowedToMeasure(kindNamespaceName) { - return time.Time{}, false - } - measurementTimestamp := time.Now() - cr.measurementsMu.Lock() - _, found := cr.measurements[kindNamespaceName] - // we only record for measurements that aren't in-progress - if !found { - cr.measurements[kindNamespaceName] = measurement{kind: kind, startTimestamp: measurementTimestamp, - expiresAt: measurementTimestamp.Add(maxMeasurementLifetime)} - } - cr.measurementsMu.Unlock() - return measurementTimestamp, !found -} - -// allowedToMeasure determines if we are allowed to measure or not. To avoid the cost of synchronisation by using locks, -// we use probability. For a value of kindNamespaceName that returns true, it will always return true. -func (cr *ConfigDurationRecorder) allowedToMeasure(kindNamespaceName string) bool { - if cr.measurementRate == 0 { - return true - } - // 1 in measurementRate chance of true - if hashToNumber(kindNamespaceName)%cr.measurementRate == 0 { - return true - } - return false -} - -func (cr *ConfigDurationRecorder) End(kind, namespace, name string) time.Time { - if !cr.enabled { - return time.Time{} - } - kindNamespaceName := fmt.Sprintf("%s/%s/%s", kind, namespace, name) - if !cr.allowedToMeasure(kindNamespaceName) { - return time.Time{} - } - measurementTimestamp := time.Now() - cr.measurementsMu.Lock() - if m, ok := cr.measurements[kindNamespaceName]; ok { - if !m.end { - m.end = true - m.endTimestamp = measurementTimestamp - cr.measurements[kindNamespaceName] = m - // if there are no OVN measurements, trigger immediate processing - if len(m.ovnMeasurements) == 0 { - select { - case cr.triggerProcessCh <- kindNamespaceName: - default: - // doesn't matter if channel is full because the measurement will be processed later anyway - } - } - } - } else { - // This can happen if Start was rejected for a resource because a measurement was in-progress for this - // kind/namespace/name, but during execution of this resource, the measurement was completed and now no record - // is found. - measurementTimestamp = time.Time{} - } - cr.measurementsMu.Unlock() - return measurementTimestamp -} - -// AddOVN adds OVN config duration to an existing recording - previously started by calling function Start -// It will return ovsdb operations which a user can add to existing operations they wish to track. -// Upon successful transaction of the operations to the ovsdb server, the user of this function must call a call-back -// function to lock-in the request to measure and report. Failure to call the call-back function, will result in no OVN -// measurement and no metrics are reported. AddOVN will result in a no-op if Start isn't called previously for the same -// kind/namespace/name. -// If multiple AddOVN is called between Start and End for the same kind/namespace/name, then the -// OVN durations will be summed and added to the total. There is an assumption that processing of kind/namespace/name is -// sequential -func (cr *ConfigDurationRecorder) AddOVN(nbClient libovsdbclient.Client, kind, namespace, name string) ( - []ovsdb.Operation, func(), time.Time, error) { - if !cr.enabled { - return []ovsdb.Operation{}, func() {}, time.Time{}, nil - } - kindNamespaceName := fmt.Sprintf("%s/%s/%s", kind, namespace, name) - if !cr.allowedToMeasure(kindNamespaceName) { - return []ovsdb.Operation{}, func() {}, time.Time{}, nil - } - cr.measurementsMu.RLock() - m, ok := cr.measurements[kindNamespaceName] - cr.measurementsMu.RUnlock() - if !ok { - // no measurement found, therefore no-op - return []ovsdb.Operation{}, func() {}, time.Time{}, nil - } - if m.end { - // existing measurement in-progress and not processed yet, therefore no-op - return []ovsdb.Operation{}, func() {}, time.Time{}, nil - } - nbGlobal := &nbdb.NBGlobal{} - nbGlobal, err := libovsdbops.GetNBGlobal(nbClient, nbGlobal) - if err != nil { - return []ovsdb.Operation{}, func() {}, time.Time{}, fmt.Errorf("failed to find OVN Northbound NB_Global table"+ - " entry: %v", err) - } - if nbGlobal.NbCfg < 0 { - return []ovsdb.Operation{}, func() {}, time.Time{}, fmt.Errorf("nb_cfg is negative, failed to add OVN measurement") - } - //stop recording if we are close to overflow - if nbGlobal.NbCfg > maxNbCfg { - return []ovsdb.Operation{}, func() {}, time.Time{}, fmt.Errorf("unable to measure OVN due to nb_cfg being close to overflow") - } - ops, err := nbClient.Where(nbGlobal).Mutate(nbGlobal, model.Mutation{ - Field: &nbGlobal.NbCfg, - Mutator: ovsdb.MutateOperationAdd, - Value: 1, - }) - if err != nil { - return []ovsdb.Operation{}, func() {}, time.Time{}, fmt.Errorf("failed to create update operation: %v", err) - } - ovnStartTimestamp := time.Now() - - return ops, func() { - // there can be a race condition here where we queue the wrong nbCfg value, but it is ok as long as it is - // less than or equal the hv_cfg value we see and this is the case because of atomic increments for nb_cfg - cr.measurementsMu.Lock() - m, ok = cr.measurements[kindNamespaceName] - if !ok { - klog.Errorf("Config duration recorder: expected a measurement entry. Call Start before AddOVN"+ - " for %s", kindNamespaceName) - cr.measurementsMu.Unlock() - return - } - m.ovnMeasurements = append(m.ovnMeasurements, ovnMeasurement{startTimestamp: ovnStartTimestamp, - nbCfg: nbGlobal.NbCfg + 1}) - cr.measurements[kindNamespaceName] = m - cr.measurementsMu.Unlock() - }, ovnStartTimestamp, nil -} - -// runMeasurementRateAdjuster will adjust the rate of measurements based on the number of nodes in the cluster and arg k -func (cr *ConfigDurationRecorder) runMeasurementRateAdjuster(kube kube.Interface, k float64, nodeCheckPeriod time.Duration, - stop <-chan struct{}) { - var currentMeasurementRate, newMeasurementRate uint64 - - updateMeasurementRate := func() { - if nodeCount, err := getNodeCount(kube); err != nil { - klog.Errorf("Config duration recorder: failed to update ticker duration considering node count: %v", err) - } else { - newMeasurementRate = uint64(math.Round(k * float64(nodeCount))) - if newMeasurementRate != currentMeasurementRate { - if newMeasurementRate > 0 { - currentMeasurementRate = newMeasurementRate - cr.measurementRate = newMeasurementRate - } - klog.V(5).Infof("Config duration recorder: updated measurement rate to approx 1 in"+ - " every %d requests", newMeasurementRate) - } - } - } - - // initial measurement rate adjustment - updateMeasurementRate() - - go func() { - nodeCheckTicker := time.NewTicker(nodeCheckPeriod) - for { - select { - case <-nodeCheckTicker.C: - updateMeasurementRate() - case <-stop: - nodeCheckTicker.Stop() - return - } - } - }() -} - -// processMeasurements manages the measurements map. It calculates metrics and cleans up finished or stale measurements -func (cr *ConfigDurationRecorder) processMeasurements(period time.Duration, updateOVNMeasurementCh chan hvCfgUpdate, - deleteOVNMeasurementCh chan int, stop <-chan struct{}) { - ticker := time.NewTicker(period) - var ovnKDelta, ovnDelta float64 - - for { - select { - case <-stop: - ticker.Stop() - return - // remove measurements if channel updateOVNMeasurementCh overflows, therefore we cannot trust existing measurements - case hvCfg := <-deleteOVNMeasurementCh: - cr.measurementsMu.Lock() - removeOVNMeasurements(cr.measurements, hvCfg) - cr.measurementsMu.Unlock() - case h := <-updateOVNMeasurementCh: - cr.measurementsMu.Lock() - cr.addHvCfg(h.hvCfg, h.timestamp) - cr.measurementsMu.Unlock() - // used for processing measurements that didn't require OVN measurement. Helps to keep measurement map small - case kindNamespaceName := <-cr.triggerProcessCh: - cr.measurementsMu.Lock() - m, ok := cr.measurements[kindNamespaceName] - if !ok { - klog.Errorf("Config duration recorder: expected measurement, but not found") - cr.measurementsMu.Unlock() - continue - } - if !m.end { - cr.measurementsMu.Unlock() - continue - } - if len(m.ovnMeasurements) != 0 { - cr.measurementsMu.Unlock() - continue - } - ovnKDelta = m.endTimestamp.Sub(m.startTimestamp).Seconds() - metricNetworkProgramming.With(prometheus.Labels{"kind": m.kind}).Observe(ovnKDelta) - klog.V(5).Infof("Config duration recorder: kind/namespace/name %s. OVN-Kubernetes controller took %v"+ - " seconds. No OVN measurement.", kindNamespaceName, ovnKDelta) - delete(cr.measurements, kindNamespaceName) - cr.measurementsMu.Unlock() - // used for processing measurements that require OVN measurement or do not or are expired. - case <-ticker.C: - start := time.Now() - cr.measurementsMu.Lock() - // process and clean up measurements - for kindNamespaceName, m := range cr.measurements { - if start.After(m.expiresAt) { - // measurement may expire if OVN is degraded or End wasn't called - klog.Warningf("Config duration recorder: measurement expired for %s", kindNamespaceName) - delete(cr.measurements, kindNamespaceName) - continue - } - if !m.end { - // measurement didn't end yet, process later - continue - } - // for when no ovn measurements requested - if len(m.ovnMeasurements) == 0 { - ovnKDelta = m.endTimestamp.Sub(m.startTimestamp).Seconds() - metricNetworkProgramming.With(prometheus.Labels{"kind": m.kind}).Observe(ovnKDelta) - klog.V(5).Infof("Config duration recorder: kind/namespace/name %s. OVN-Kubernetes controller"+ - " took %v seconds. No OVN measurement.", kindNamespaceName, ovnKDelta) - delete(cr.measurements, kindNamespaceName) - continue - } - // for each kind/namespace/name, there can be multiple calls to AddOVN between start and end - // we sum all the OVN durations and add it to the start and end duration - // first lets make sure all OVN measurements are finished - if complete := allOVNMeasurementsComplete(m.ovnMeasurements); !complete { - continue - } - - ovnKDelta = m.endTimestamp.Sub(m.startTimestamp).Seconds() - ovnDelta = calculateOVNDuration(m.ovnMeasurements) - metricNetworkProgramming.With(prometheus.Labels{"kind": m.kind}).Observe(ovnKDelta + ovnDelta) - metricNetworkProgrammingOVN.Observe(ovnDelta) - klog.V(5).Infof("Config duration recorder: kind/namespace/name %s. OVN-Kubernetes controller took"+ - " %v seconds. OVN took %v seconds. Total took %v seconds", kindNamespaceName, ovnKDelta, - ovnDelta, ovnDelta+ovnKDelta) - delete(cr.measurements, kindNamespaceName) - } - cr.measurementsMu.Unlock() - } - } -} - -func (cr *ConfigDurationRecorder) addHvCfg(hvCfg, hvCfgTimestamp int) { - var altered bool - for i, m := range cr.measurements { - altered = false - for iOvnM, ovnM := range m.ovnMeasurements { - if ovnM.complete { - continue - } - if ovnM.nbCfg <= hvCfg { - ovnM.endTimestamp = time.UnixMilli(int64(hvCfgTimestamp)) - ovnM.complete = true - m.ovnMeasurements[iOvnM] = ovnM - altered = true - } - } - if altered { - cr.measurements[i] = m - } - } -} - -// removeOVNMeasurements remove any OVN measurements less than or equal argument hvCfg -func removeOVNMeasurements(measurements map[string]measurement, hvCfg int) { - for kindNamespaceName, m := range measurements { - var indexToDelete []int - for i, ovnM := range m.ovnMeasurements { - if ovnM.nbCfg <= hvCfg { - indexToDelete = append(indexToDelete, i) - } - } - if len(indexToDelete) == 0 { - continue - } - if len(indexToDelete) == len(m.ovnMeasurements) { - delete(measurements, kindNamespaceName) - } - for _, iDel := range indexToDelete { - m.ovnMeasurements = removeOVNMeasurement(m.ovnMeasurements, iDel) - } - measurements[kindNamespaceName] = m - } -} - -func removeOVNMeasurement(oM []ovnMeasurement, i int) []ovnMeasurement { - oM[i] = oM[len(oM)-1] - return oM[:len(oM)-1] -} -func hashToNumber(s string) uint64 { - h := fnv.New64() - h.Write([]byte(s)) - return h.Sum64() -} - -func calculateOVNDuration(ovnMeasurements []ovnMeasurement) float64 { - var totalDuration float64 - for _, oM := range ovnMeasurements { - if !oM.complete { - continue - } - totalDuration += oM.endTimestamp.Sub(oM.startTimestamp).Seconds() - } - return totalDuration -} - -func allOVNMeasurementsComplete(ovnMeasurements []ovnMeasurement) bool { - for _, oM := range ovnMeasurements { - if !oM.complete { - return false - } - } - return true -} - -// merge direct copy from k8 pkg/proxy/metrics/metrics.go -func merge(slices ...[]float64) []float64 { - result := make([]float64, 1) - for _, s := range slices { - result = append(result, s...) - } - return result -} - -func getNodeCount(kube kube.Interface) (int, error) { - nodes, err := kube.GetNodes() - if err != nil { - return 0, fmt.Errorf("unable to retrieve node list: %v", err) - } - return len(nodes), nil -} - // setNbE2eTimestamp return true if setting timestamp to NB global options is successful func setNbE2eTimestamp(ovnNBClient libovsdbclient.Client, timestamp int64) bool { // assumption that only first row is relevant in NB_Global table diff --git a/go-controller/pkg/metrics/ovs.go b/go-controller/pkg/metrics/ovs.go index b2cc1403a0..455142ae6b 100644 --- a/go-controller/pkg/metrics/ovs.go +++ b/go-controller/pkg/metrics/ovs.go @@ -18,6 +18,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" ovsops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops/ovs" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -27,15 +28,15 @@ var ( // ovs datapath Metrics var metricOvsDpTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp_total", Help: "Represents total number of datapaths on the system.", }) var metricOvsDp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp", Help: "A metric with a constant '1' value labeled by datapath " + "name present on the instance."}, @@ -46,8 +47,8 @@ var metricOvsDp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsDpIfTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp_if_total", Help: "Represents the number of ports connected to the datapath."}, []string{ @@ -56,8 +57,8 @@ var metricOvsDpIfTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsDpFlowsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp_flows_total", Help: "Represents the number of flows in datapath."}, []string{ @@ -66,8 +67,8 @@ var metricOvsDpFlowsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsDpFlowsLookupHit = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp_flows_lookup_hit", Help: "Represents number of packets matching the existing flows " + "while processing incoming packets in the datapath."}, @@ -77,8 +78,8 @@ var metricOvsDpFlowsLookupHit = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsDpFlowsLookupMissed = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp_flows_lookup_missed", Help: "Represents the number of packets not matching any existing " + "flow and require user space processing."}, @@ -88,8 +89,8 @@ var metricOvsDpFlowsLookupMissed = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsDpFlowsLookupLost = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp_flows_lookup_lost", Help: "number of packets destined for user space process but " + "subsequently dropped before reaching userspace."}, @@ -99,8 +100,8 @@ var metricOvsDpFlowsLookupLost = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsDpPacketsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp_packets_total", Help: "Represents the total number of packets datapath processed " + "which is the sum of hit and missed."}, @@ -110,8 +111,8 @@ var metricOvsDpPacketsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsdpMasksHit = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp_masks_hit", Help: "Represents the total number of masks visited for matching incoming packets.", }, @@ -121,8 +122,8 @@ var metricOvsdpMasksHit = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsDpMasksTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp_masks_total", Help: "Represents the number of masks in a datapath."}, []string{ @@ -131,8 +132,8 @@ var metricOvsDpMasksTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsDpMasksHitRatio = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "dp_masks_hit_ratio", Help: "Represents the average number of masks visited per packet " + "the ratio between hit and total number of packets processed by the datapath."}, @@ -143,16 +144,16 @@ var metricOvsDpMasksHitRatio = prometheus.NewGaugeVec(prometheus.GaugeOpts{ // ovs bridge statistics & attributes metrics var metricOvsBridgeTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "bridge_total", Help: "Represents total number of OVS bridges on the system.", }, ) var metricOvsBridge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "bridge", Help: "A metric with a constant '1' value labeled by bridge name " + "present on the instance."}, @@ -162,8 +163,8 @@ var metricOvsBridge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsBridgePortsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "bridge_ports_total", Help: "Represents the number of OVS ports on the bridge."}, []string{ @@ -172,8 +173,8 @@ var metricOvsBridgePortsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ) var metricOvsBridgeFlowsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "bridge_flows_total", Help: "Represents the number of OpenFlow flows on the OVS bridge."}, []string{ @@ -183,57 +184,57 @@ var metricOvsBridgeFlowsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ // ovs interface metrics var metricOvsInterfaceResetsTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "interface_resets_total", Help: "The number of link state changes observed by Open vSwitch interface(s).", }) var metricOvsInterfaceRxDroppedTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "interface_rx_dropped_total", Help: "The total number of received packets dropped by Open vSwitch interface(s).", }) var metricOvsInterfaceTxDroppedTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "interface_tx_dropped_total", Help: "The total number of transmitted packets dropped by Open vSwitch interface(s).", }) var metricOvsInterfaceRxErrorsTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "interface_rx_errors_total", Help: "The total number of received packets with errors by Open vSwitch interface(s).", }) var metricOvsInterfaceTxErrorsTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "interface_tx_errors_total", Help: "The total number of transmitted packets with errors by Open vSwitch interface(s).", }) var metricOvsInterfaceCollisionsTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "interface_collisions_total", Help: "The total number of packet collisions transmitted by Open vSwitch interface(s).", }) var metricOvsInterfaceTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "interfaces_total", Help: "The total number of Open vSwitch interface(s) created for pods", }) var MetricOvsInterfaceUpWait = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "interface_up_wait_seconds_total", Help: "The total number of seconds that is required to wait for pod " + "Open vSwitch interface until its available", @@ -241,16 +242,16 @@ var MetricOvsInterfaceUpWait = prometheus.NewCounter(prometheus.CounterOpts{ // ovs memory metrics var metricOvsHandlersTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "handlers_total", Help: "Represents the number of handlers thread. This thread reads upcalls from dpif, " + "forwards each upcall's packet and possibly sets up a kernel flow as a cache.", }) var metricOvsRevalidatorsTotal = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "revalidators_total", Help: "Represents the number of revalidators thread. This thread processes datapath flows, " + "updates OpenFlow statistics, and updates or removes them if necessary.", @@ -258,16 +259,16 @@ var metricOvsRevalidatorsTotal = prometheus.NewGauge(prometheus.GaugeOpts{ // ovs Hw offload metrics var metricOvsHwOffload = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "hw_offload", Help: "Represents whether netdev flow offload to hardware is enabled " + "or not -- false(0) and true(1).", }) var metricOvsTcPolicy = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, - Subsystem: MetricOvsSubsystemVswitchd, + Namespace: types.MetricOvsNamespace, + Subsystem: types.MetricOvsSubsystemVswitchd, Name: "tc_policy", Help: "Represents the policy used with HW offloading " + "-- none(0), skip_sw(1), and skip_hw(2).", @@ -310,15 +311,15 @@ func ovsDatapathLookupsMetrics(output, datapath string) { } switch elem[0] { case "hit": - value := parseMetricToFloat(MetricOvsSubsystemVswitchd, "dp_flows_lookup_hit", elem[1]) + value := parseMetricToFloat(types.MetricOvsSubsystemVswitchd, "dp_flows_lookup_hit", elem[1]) datapathPacketsTotal += value metricOvsDpFlowsLookupHit.WithLabelValues(datapath).Set(value) case "missed": - value := parseMetricToFloat(MetricOvsSubsystemVswitchd, "dp_flows_lookup_missed", elem[1]) + value := parseMetricToFloat(types.MetricOvsSubsystemVswitchd, "dp_flows_lookup_missed", elem[1]) datapathPacketsTotal += value metricOvsDpFlowsLookupMissed.WithLabelValues(datapath).Set(value) case "lost": - value := parseMetricToFloat(MetricOvsSubsystemVswitchd, "dp_flows_lookup_lost", elem[1]) + value := parseMetricToFloat(types.MetricOvsSubsystemVswitchd, "dp_flows_lookup_lost", elem[1]) metricOvsDpFlowsLookupLost.WithLabelValues(datapath).Set(value) } } @@ -335,13 +336,13 @@ func ovsDatapathMasksMetrics(output, datapath string) { } switch elem[0] { case "hit": - value := parseMetricToFloat(MetricOvsSubsystemVswitchd, "dp_masks_hit", elem[1]) + value := parseMetricToFloat(types.MetricOvsSubsystemVswitchd, "dp_masks_hit", elem[1]) metricOvsdpMasksHit.WithLabelValues(datapath).Set(value) case "total": - value := parseMetricToFloat(MetricOvsSubsystemVswitchd, "dp_masks_total", elem[1]) + value := parseMetricToFloat(types.MetricOvsSubsystemVswitchd, "dp_masks_total", elem[1]) metricOvsDpMasksTotal.WithLabelValues(datapath).Set(value) case "hit/pkt": - value := parseMetricToFloat(MetricOvsSubsystemVswitchd, "dp_masks_hit_ratio", elem[1]) + value := parseMetricToFloat(types.MetricOvsSubsystemVswitchd, "dp_masks_hit_ratio", elem[1]) metricOvsDpMasksHitRatio.WithLabelValues(datapath).Set(value) } } @@ -419,7 +420,7 @@ func setOvsDatapathMetrics(ovsAppctl ovsClient, datapaths []string) (err error) datapathPortCount++ } else if strings.HasPrefix(output, "flows:") { flowFields := strings.Fields(output) - value := parseMetricToFloat(MetricOvsSubsystemVswitchd, "dp_flows_total", flowFields[1]) + value := parseMetricToFloat(types.MetricOvsSubsystemVswitchd, "dp_flows_total", flowFields[1]) metricOvsDpFlowsTotal.WithLabelValues(datapathName).Set(value) } } @@ -504,7 +505,7 @@ func getOvsBridgeOpenFlowsCount(ovsOfctl ovsClient, bridgeName string) (float64, if strings.HasPrefix(kvPair, "flow_count=") { value := strings.Split(kvPair, "=")[1] metricName := bridgeName + "flows_total" - return parseMetricToFloat(MetricOvsSubsystemVswitchd, metricName, value), nil + return parseMetricToFloat(types.MetricOvsSubsystemVswitchd, metricName, value), nil } } return 0, fmt.Errorf("ovs-ofctl dump-aggregate %s output didn't contain "+ @@ -595,11 +596,11 @@ func setOvsMemoryMetrics(ovsVswitchdAppctl ovsClient) (err error) { for _, kvPair := range strings.Fields(stdout) { if strings.HasPrefix(kvPair, "handlers:") { value := strings.Split(kvPair, ":")[1] - count := parseMetricToFloat(MetricOvsSubsystemVswitchd, "handlers_total", value) + count := parseMetricToFloat(types.MetricOvsSubsystemVswitchd, "handlers_total", value) metricOvsHandlersTotal.Set(count) } else if strings.HasPrefix(kvPair, "revalidators:") { value := strings.Split(kvPair, ":")[1] - count := parseMetricToFloat(MetricOvsSubsystemVswitchd, "revalidators_total", value) + count := parseMetricToFloat(types.MetricOvsSubsystemVswitchd, "revalidators_total", value) metricOvsRevalidatorsTotal.Set(count) } } @@ -846,7 +847,7 @@ func registerOvsMetrics(ovsDBClient libovsdbclient.Client, metricsScrapeInterval getOvsVersionInfo(ovsDBClient) registry.MustRegister(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ - Namespace: MetricOvsNamespace, + Namespace: types.MetricOvsNamespace, Name: "build_info", Help: "A metric with a constant '1' value labeled by ovs version.", ConstLabels: prometheus.Labels{ @@ -890,18 +891,18 @@ func registerOvsMetrics(ovsDBClient libovsdbclient.Client, metricsScrapeInterval registry.MustRegister(MetricOvsInterfaceUpWait) // Register the OVS coverage/show metrics componentCoverageShowMetricsMap[ovsVswitchd] = ovsVswitchdCoverageShowMetricsMap - registerCoverageShowMetrics(ovsVswitchd, MetricOvsNamespace, MetricOvsSubsystemVswitchd) + registerCoverageShowMetrics(ovsVswitchd, types.MetricOvsNamespace, types.MetricOvsSubsystemVswitchd) // When ovnkube-node is running in privileged mode, the hostPID will be set to true, // and therefore it can monitor OVS running on the host using PID. if !config.UnprivilegedMode { registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{ PidFn: prometheus.NewPidFileFn("/var/run/openvswitch/ovs-vswitchd.pid"), - Namespace: fmt.Sprintf("%s_%s", MetricOvsNamespace, MetricOvsSubsystemVswitchd), + Namespace: fmt.Sprintf("%s_%s", types.MetricOvsNamespace, types.MetricOvsSubsystemVswitchd), })) registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{ PidFn: prometheus.NewPidFileFn("/var/run/openvswitch/ovsdb-server.pid"), - Namespace: fmt.Sprintf("%s_%s", MetricOvsNamespace, MetricOvsSubsystemDB), + Namespace: fmt.Sprintf("%s_%s", types.MetricOvsNamespace, types.MetricOvsSubsystemDB), })) } diff --git a/go-controller/pkg/metrics/recorders/duration.go b/go-controller/pkg/metrics/recorders/duration.go new file mode 100644 index 0000000000..c0ae704e8f --- /dev/null +++ b/go-controller/pkg/metrics/recorders/duration.go @@ -0,0 +1,565 @@ +package recorders + +import ( + "fmt" + "hash/fnv" + "math" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "k8s.io/klog/v2" + + "github.com/ovn-org/libovsdb/cache" + libovsdbclient "github.com/ovn-org/libovsdb/client" + "github.com/ovn-org/libovsdb/model" + "github.com/ovn-org/libovsdb/ovsdb" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" +) + +const ( + updateOVNMeasurementChSize = 500 + deleteOVNMeasurementChSize = 50 + processChSize = 1000 + nbGlobalTable = "NB_Global" + //fixme: remove when bug is fixed in OVN (Red Hat bugzilla bug number 2074019). Also, handle overflow event. + maxNbCfg = math.MaxUint32 - 1000 + maxMeasurementLifetime = 20 * time.Minute +) + +var configDurationRegOnce sync.Once + +type ConfigDurationRecorder struct { + // rate at which measurements are allowed. Probabilistically, 1 in every measurementRate + measurementRate uint64 + measurements map[string]measurement + // controls RW access to measurements map + measurementsMu sync.RWMutex + // channel to trigger processing a measurement following call to End func. Channel string is kind/namespace/name + triggerProcessCh chan string + enabled bool +} + +type ovnMeasurement struct { + // time just before ovsdb tx is called + startTimestamp time.Time + // time when the nbCfg value and its associated configuration is applied to all nodes + endTimestamp time.Time + // OVN measurement complete - start and end timestamps are valid + complete bool + // nb_cfg value that started the measurement + nbCfg int +} + +// measurement stores a measurement attempt through OVN-Kubernetes controller and optionally OVN +type measurement struct { + // kubernetes kind e.g. pod or service + kind string + // time when Add is executed + startTimestamp time.Time + // time when End is executed + endTimestamp time.Time + // if true endTimestamp is valid + end bool + // time when this measurement expires. Set during Add + expiresAt time.Time + // OVN measurement(s) via AddOVN + ovnMeasurements []ovnMeasurement +} + +// hvCfgUpdate holds the information received from OVN Northbound event handler +type hvCfgUpdate struct { + // timestamp is in milliseconds + timestamp int + hvCfg int +} + +// global variable is needed because this functionality is accessed in many functions +var cdr *ConfigDurationRecorder + +// lock for accessing the cdr global variable +var cdrMutex sync.Mutex + +func GetConfigDurationRecorder() *ConfigDurationRecorder { + cdrMutex.Lock() + defer cdrMutex.Unlock() + if cdr == nil { + cdr = &ConfigDurationRecorder{} + } + return cdr +} + +// removeOVNMeasurements remove any OVN measurements less than or equal argument hvCfg +func removeOVNMeasurements(measurements map[string]measurement, hvCfg int) { + for kindNamespaceName, m := range measurements { + var indexToDelete []int + for i, ovnM := range m.ovnMeasurements { + if ovnM.nbCfg <= hvCfg { + indexToDelete = append(indexToDelete, i) + } + } + if len(indexToDelete) == 0 { + continue + } + if len(indexToDelete) == len(m.ovnMeasurements) { + delete(measurements, kindNamespaceName) + } + for _, iDel := range indexToDelete { + m.ovnMeasurements = removeOVNMeasurement(m.ovnMeasurements, iDel) + } + measurements[kindNamespaceName] = m + } +} + +var metricNetworkProgramming prometheus.ObserverVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, + Name: "network_programming_duration_seconds", + Help: "The duration to apply network configuration for a kind (e.g. pod, service, networkpolicy). " + + "Configuration includes add, update and delete events for each kind.", + Buckets: merge( + prometheus.LinearBuckets(0.25, 0.25, 2), // 0.25s, 0.50s + prometheus.LinearBuckets(1, 1, 59), // 1s, 2s, 3s, ... 59s + prometheus.LinearBuckets(60, 5, 12), // 60s, 65s, 70s, ... 115s + prometheus.LinearBuckets(120, 30, 11))}, // 2min, 2.5min, 3min, ..., 7min + []string{ + "kind", + }) + +var metricNetworkProgrammingOVN = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, + Name: "network_programming_ovn_duration_seconds", + Help: "The duration for OVN to apply network configuration", + Buckets: merge( + prometheus.LinearBuckets(0.25, 0.25, 2), // 0.25s, 0.50s + prometheus.LinearBuckets(1, 1, 59), // 1s, 2s, 3s, ... 59s + prometheus.LinearBuckets(60, 5, 12), // 60s, 65s, 70s, ... 115s + prometheus.LinearBuckets(120, 30, 11))}, // 2min, 2.5min, 3min, ..., 7min +) + +// Run monitors the config duration for OVN-Kube master to configure k8 kinds. A measurement maybe allowed and this is +// related to the number of k8 nodes, N [1] and by argument k [2] where there is a probability that 1 out of N*k +// measurement attempts are allowed. If k=0, all measurements are allowed. mUpdatePeriod determines the period to +// process and publish metrics +// [1] 1 0. The measurement rate is proportional to + // the number of nodes, N and argument k. 1 out of every N*k attempted measurements will succeed. + + // For the optional OVN measurement by calling AddOVN, when the CMS is about to make a transaction to configure + // whatever kind, a call to AddOVN function allows the caller to measure OVN duration. + // An ovsdb operation is returned to the caller of AddOVN, which they can bundle with their existing transactions + // sent to OVN which will tell OVN to measure how long it takes to configure all nodes with the config in the transaction. + // Config duration then waits for OVN to configure all nodes and calculates the time delta. + + // ** configuration duration recorder - caveats ** + // For the optional OVN recording, it does not give you an exact time duration for how long it takes to configure your + // k8 kind. When you are recording how long it takes OVN to complete your configuration to all nodes, other + // transactions may have occurred which may increases the overall time. You may also get longer processing times if one + // or more nodes are unavailable because we are measuring how long the functionality takes to apply to ALL nodes. + + // ** configuration duration recorder - How the duration of the config is measured within OVN ** + // We increment the nb_cfg integer value in the NB_Global table. + // ovn-northd notices the nb_cfg change and copies the nb_cfg value to SB_Global table field nb_cfg along with any + // other configuration that is changed in OVN Northbound database. + // All ovn-controllers detect nb_cfg value change and generate a 'barrier' on the openflow connection to the + // nodes ovs-vswitchd. Once ovn-controllers receive the 'barrier processed' reply from ovs-vswitchd which + // indicates that all relevant openflow operations associated with NB_Globals nb_cfg value have been + // propagated to the nodes OVS, it copies the SB_Global nb_cfg value to its Chassis_Private table nb_cfg record. + // ovn-northd detects changes to the Chassis_Private startRecords and computes the minimum nb_cfg for all Chassis_Private + // nb_cfg and stores this in NB_Global hv_cfg field along with a timestamp to field hv_cfg_timestamp which + // reflects the time when the slowest chassis catches up with the northbound configuration. + configDurationRegOnce.Do(func() { + prometheus.MustRegister(metricNetworkProgramming) + prometheus.MustRegister(metricNetworkProgrammingOVN) + }) + + cr.measurements = make(map[string]measurement) + // watch node count and adjust measurement rate if node count changes + cr.runMeasurementRateAdjuster(wf, k, time.Hour, stop) + // we currently do not clean the following channels up upon exit + cr.triggerProcessCh = make(chan string, processChSize) + updateOVNMeasurementCh := make(chan hvCfgUpdate, updateOVNMeasurementChSize) + deleteOVNMeasurementCh := make(chan int, deleteOVNMeasurementChSize) + go cr.processMeasurements(workerLoopPeriod, updateOVNMeasurementCh, deleteOVNMeasurementCh, stop) + + nbClient.Cache().AddEventHandler(&cache.EventHandlerFuncs{ + UpdateFunc: func(table string, old model.Model, new model.Model) { + if table != nbGlobalTable { + return + } + oldRow := old.(*nbdb.NBGlobal) + newRow := new.(*nbdb.NBGlobal) + + if oldRow.HvCfg != newRow.HvCfg && oldRow.HvCfgTimestamp != newRow.HvCfgTimestamp && newRow.HvCfgTimestamp > 0 { + select { + case updateOVNMeasurementCh <- hvCfgUpdate{hvCfg: newRow.HvCfg, timestamp: newRow.HvCfgTimestamp}: + default: + klog.Warning("Config duration recorder: unable to update OVN measurement") + select { + case deleteOVNMeasurementCh <- newRow.HvCfg: + default: + } + } + } + }, + }) + cr.enabled = true +} + +// Start allows the caller to attempt measurement of a control plane configuration duration, as a metric, +// the duration between functions Start and End. Optionally, if you wish to record OVN config duration, +// call AddOVN which will add the duration for OVN to apply the configuration to all nodes. +// The caller must pass kind,namespace,name which will be used to determine if the object +// is allowed to record. To allow no locking, each go routine that calls this function, can determine itself +// if it is allowed to measure. +// There is a mandatory two-step process to complete a measurement. +// Step 1) Call Start when you wish to begin a measurement - ideally when processing for the object starts +// Step 2) Call End which will complete a measurement +// Optionally, call AddOVN when you are making a transaction to OVN in order to add on the OVN duration to an existing +// measurement. This must be called between Start and End. Not every call to Start will result in a measurement +// and the rate of measurements depends on the number of nodes and function Run arg k. +// Only one measurement for a kind/namespace/name is allowed until the current measurement is Ended (via End) and +// processed. This is guaranteed by workqueues (even with multiple workers) and informer event handlers. +func (cr *ConfigDurationRecorder) Start(kind, namespace, name string) (time.Time, bool) { + if !cr.enabled { + return time.Time{}, false + } + kindNamespaceName := fmt.Sprintf("%s/%s/%s", kind, namespace, name) + if !cr.allowedToMeasure(kindNamespaceName) { + return time.Time{}, false + } + measurementTimestamp := time.Now() + cr.measurementsMu.Lock() + _, found := cr.measurements[kindNamespaceName] + // we only record for measurements that aren't in-progress + if !found { + cr.measurements[kindNamespaceName] = measurement{kind: kind, startTimestamp: measurementTimestamp, + expiresAt: measurementTimestamp.Add(maxMeasurementLifetime)} + } + cr.measurementsMu.Unlock() + return measurementTimestamp, !found +} + +// allowedToMeasure determines if we are allowed to measure or not. To avoid the cost of synchronisation by using locks, +// we use probability. For a value of kindNamespaceName that returns true, it will always return true. +func (cr *ConfigDurationRecorder) allowedToMeasure(kindNamespaceName string) bool { + if cr.measurementRate == 0 { + return true + } + // 1 in measurementRate chance of true + if hashToNumber(kindNamespaceName)%cr.measurementRate == 0 { + return true + } + return false +} + +func (cr *ConfigDurationRecorder) End(kind, namespace, name string) time.Time { + if !cr.enabled { + return time.Time{} + } + kindNamespaceName := fmt.Sprintf("%s/%s/%s", kind, namespace, name) + if !cr.allowedToMeasure(kindNamespaceName) { + return time.Time{} + } + measurementTimestamp := time.Now() + cr.measurementsMu.Lock() + if m, ok := cr.measurements[kindNamespaceName]; ok { + if !m.end { + m.end = true + m.endTimestamp = measurementTimestamp + cr.measurements[kindNamespaceName] = m + // if there are no OVN measurements, trigger immediate processing + if len(m.ovnMeasurements) == 0 { + select { + case cr.triggerProcessCh <- kindNamespaceName: + default: + // doesn't matter if channel is full because the measurement will be processed later anyway + } + } + } + } else { + // This can happen if Start was rejected for a resource because a measurement was in-progress for this + // kind/namespace/name, but during execution of this resource, the measurement was completed and now no record + // is found. + measurementTimestamp = time.Time{} + } + cr.measurementsMu.Unlock() + return measurementTimestamp +} + +// AddOVN adds OVN config duration to an existing recording - previously started by calling function Start +// It will return ovsdb operations which a user can add to existing operations they wish to track. +// Upon successful transaction of the operations to the ovsdb server, the user of this function must call a call-back +// function to lock-in the request to measure and report. Failure to call the call-back function, will result in no OVN +// measurement and no metrics are reported. AddOVN will result in a no-op if Start isn't called previously for the same +// kind/namespace/name. +// If multiple AddOVN is called between Start and End for the same kind/namespace/name, then the +// OVN durations will be summed and added to the total. There is an assumption that processing of kind/namespace/name is +// sequential +func (cr *ConfigDurationRecorder) AddOVN(nbClient libovsdbclient.Client, kind, namespace, name string) ( + []ovsdb.Operation, func(), time.Time, error) { + if !cr.enabled { + return []ovsdb.Operation{}, func() {}, time.Time{}, nil + } + kindNamespaceName := fmt.Sprintf("%s/%s/%s", kind, namespace, name) + if !cr.allowedToMeasure(kindNamespaceName) { + return []ovsdb.Operation{}, func() {}, time.Time{}, nil + } + cr.measurementsMu.RLock() + m, ok := cr.measurements[kindNamespaceName] + cr.measurementsMu.RUnlock() + if !ok { + // no measurement found, therefore no-op + return []ovsdb.Operation{}, func() {}, time.Time{}, nil + } + if m.end { + // existing measurement in-progress and not processed yet, therefore no-op + return []ovsdb.Operation{}, func() {}, time.Time{}, nil + } + nbGlobal := &nbdb.NBGlobal{} + nbGlobal, err := libovsdbops.GetNBGlobal(nbClient, nbGlobal) + if err != nil { + return []ovsdb.Operation{}, func() {}, time.Time{}, fmt.Errorf("failed to find OVN Northbound NB_Global table"+ + " entry: %v", err) + } + if nbGlobal.NbCfg < 0 { + return []ovsdb.Operation{}, func() {}, time.Time{}, fmt.Errorf("nb_cfg is negative, failed to add OVN measurement") + } + //stop recording if we are close to overflow + if nbGlobal.NbCfg > maxNbCfg { + return []ovsdb.Operation{}, func() {}, time.Time{}, fmt.Errorf("unable to measure OVN due to nb_cfg being close to overflow") + } + ops, err := nbClient.Where(nbGlobal).Mutate(nbGlobal, model.Mutation{ + Field: &nbGlobal.NbCfg, + Mutator: ovsdb.MutateOperationAdd, + Value: 1, + }) + if err != nil { + return []ovsdb.Operation{}, func() {}, time.Time{}, fmt.Errorf("failed to create update operation: %v", err) + } + ovnStartTimestamp := time.Now() + + return ops, func() { + // there can be a race condition here where we queue the wrong nbCfg value, but it is ok as long as it is + // less than or equal the hv_cfg value we see and this is the case because of atomic increments for nb_cfg + cr.measurementsMu.Lock() + m, ok = cr.measurements[kindNamespaceName] + if !ok { + klog.Errorf("Config duration recorder: expected a measurement entry. Call Start before AddOVN"+ + " for %s", kindNamespaceName) + cr.measurementsMu.Unlock() + return + } + m.ovnMeasurements = append(m.ovnMeasurements, ovnMeasurement{startTimestamp: ovnStartTimestamp, + nbCfg: nbGlobal.NbCfg + 1}) + cr.measurements[kindNamespaceName] = m + cr.measurementsMu.Unlock() + }, ovnStartTimestamp, nil +} + +// runMeasurementRateAdjuster will adjust the rate of measurements based on the number of nodes in the cluster and arg k +func (cr *ConfigDurationRecorder) runMeasurementRateAdjuster(wf *factory.WatchFactory, k float64, nodeCheckPeriod time.Duration, + stop <-chan struct{}) { + var currentMeasurementRate, newMeasurementRate uint64 + + updateMeasurementRate := func() { + if nodeCount, err := getNodeCount(wf); err != nil { + klog.Errorf("Config duration recorder: failed to update ticker duration considering node count: %v", err) + } else { + newMeasurementRate = uint64(math.Round(k * float64(nodeCount))) + if newMeasurementRate != currentMeasurementRate { + if newMeasurementRate > 0 { + currentMeasurementRate = newMeasurementRate + cr.measurementRate = newMeasurementRate + } + klog.V(5).Infof("Config duration recorder: updated measurement rate to approx 1 in"+ + " every %d requests", newMeasurementRate) + } + } + } + + // initial measurement rate adjustment + updateMeasurementRate() + + go func() { + nodeCheckTicker := time.NewTicker(nodeCheckPeriod) + for { + select { + case <-nodeCheckTicker.C: + updateMeasurementRate() + case <-stop: + nodeCheckTicker.Stop() + return + } + } + }() +} + +// processMeasurements manages the measurements map. It calculates metrics and cleans up finished or stale measurements +func (cr *ConfigDurationRecorder) processMeasurements(period time.Duration, updateOVNMeasurementCh chan hvCfgUpdate, + deleteOVNMeasurementCh chan int, stop <-chan struct{}) { + ticker := time.NewTicker(period) + var ovnKDelta, ovnDelta float64 + + for { + select { + case <-stop: + ticker.Stop() + return + // remove measurements if channel updateOVNMeasurementCh overflows, therefore we cannot trust existing measurements + case hvCfg := <-deleteOVNMeasurementCh: + cr.measurementsMu.Lock() + removeOVNMeasurements(cr.measurements, hvCfg) + cr.measurementsMu.Unlock() + case h := <-updateOVNMeasurementCh: + cr.measurementsMu.Lock() + cr.addHvCfg(h.hvCfg, h.timestamp) + cr.measurementsMu.Unlock() + // used for processing measurements that didn't require OVN measurement. Helps to keep measurement map small + case kindNamespaceName := <-cr.triggerProcessCh: + cr.measurementsMu.Lock() + m, ok := cr.measurements[kindNamespaceName] + if !ok { + klog.Errorf("Config duration recorder: expected measurement, but not found") + cr.measurementsMu.Unlock() + continue + } + if !m.end { + cr.measurementsMu.Unlock() + continue + } + if len(m.ovnMeasurements) != 0 { + cr.measurementsMu.Unlock() + continue + } + ovnKDelta = m.endTimestamp.Sub(m.startTimestamp).Seconds() + metricNetworkProgramming.With(prometheus.Labels{"kind": m.kind}).Observe(ovnKDelta) + klog.V(5).Infof("Config duration recorder: kind/namespace/name %s. OVN-Kubernetes controller took %v"+ + " seconds. No OVN measurement.", kindNamespaceName, ovnKDelta) + delete(cr.measurements, kindNamespaceName) + cr.measurementsMu.Unlock() + // used for processing measurements that require OVN measurement or do not or are expired. + case <-ticker.C: + start := time.Now() + cr.measurementsMu.Lock() + // process and clean up measurements + for kindNamespaceName, m := range cr.measurements { + if start.After(m.expiresAt) { + // measurement may expire if OVN is degraded or End wasn't called + klog.Warningf("Config duration recorder: measurement expired for %s", kindNamespaceName) + delete(cr.measurements, kindNamespaceName) + continue + } + if !m.end { + // measurement didn't end yet, process later + continue + } + // for when no ovn measurements requested + if len(m.ovnMeasurements) == 0 { + ovnKDelta = m.endTimestamp.Sub(m.startTimestamp).Seconds() + metricNetworkProgramming.With(prometheus.Labels{"kind": m.kind}).Observe(ovnKDelta) + klog.V(5).Infof("Config duration recorder: kind/namespace/name %s. OVN-Kubernetes controller"+ + " took %v seconds. No OVN measurement.", kindNamespaceName, ovnKDelta) + delete(cr.measurements, kindNamespaceName) + continue + } + // for each kind/namespace/name, there can be multiple calls to AddOVN between start and end + // we sum all the OVN durations and add it to the start and end duration + // first lets make sure all OVN measurements are finished + if complete := allOVNMeasurementsComplete(m.ovnMeasurements); !complete { + continue + } + + ovnKDelta = m.endTimestamp.Sub(m.startTimestamp).Seconds() + ovnDelta = calculateOVNDuration(m.ovnMeasurements) + metricNetworkProgramming.With(prometheus.Labels{"kind": m.kind}).Observe(ovnKDelta + ovnDelta) + metricNetworkProgrammingOVN.Observe(ovnDelta) + klog.V(5).Infof("Config duration recorder: kind/namespace/name %s. OVN-Kubernetes controller took"+ + " %v seconds. OVN took %v seconds. Total took %v seconds", kindNamespaceName, ovnKDelta, + ovnDelta, ovnDelta+ovnKDelta) + delete(cr.measurements, kindNamespaceName) + } + cr.measurementsMu.Unlock() + } + } +} + +func (cr *ConfigDurationRecorder) addHvCfg(hvCfg, hvCfgTimestamp int) { + var altered bool + for i, m := range cr.measurements { + altered = false + for iOvnM, ovnM := range m.ovnMeasurements { + if ovnM.complete { + continue + } + if ovnM.nbCfg <= hvCfg { + ovnM.endTimestamp = time.UnixMilli(int64(hvCfgTimestamp)) + ovnM.complete = true + m.ovnMeasurements[iOvnM] = ovnM + altered = true + } + } + if altered { + cr.measurements[i] = m + } + } +} + +func getNodeCount(wf *factory.WatchFactory) (int, error) { + nodes, err := wf.GetNodes() + if err != nil { + return 0, fmt.Errorf("unable to retrieve node list: %v", err) + } + return len(nodes), nil +} + +func removeOVNMeasurement(oM []ovnMeasurement, i int) []ovnMeasurement { + oM[i] = oM[len(oM)-1] + return oM[:len(oM)-1] +} +func hashToNumber(s string) uint64 { + h := fnv.New64() + h.Write([]byte(s)) + return h.Sum64() +} + +func calculateOVNDuration(ovnMeasurements []ovnMeasurement) float64 { + var totalDuration float64 + for _, oM := range ovnMeasurements { + if !oM.complete { + continue + } + totalDuration += oM.endTimestamp.Sub(oM.startTimestamp).Seconds() + } + return totalDuration +} + +func allOVNMeasurementsComplete(ovnMeasurements []ovnMeasurement) bool { + for _, oM := range ovnMeasurements { + if !oM.complete { + return false + } + } + return true +} + +// merge direct copy from k8 pkg/proxy/metrics/metrics.go +func merge(slices ...[]float64) []float64 { + result := make([]float64, 1) + for _, s := range slices { + result = append(result, s...) + } + return result +} diff --git a/go-controller/pkg/metrics/ovnkube_controller_test.go b/go-controller/pkg/metrics/recorders/duration_test.go similarity index 86% rename from go-controller/pkg/metrics/ovnkube_controller_test.go rename to go-controller/pkg/metrics/recorders/duration_test.go index 1ff008db59..ee436d8d1b 100644 --- a/go-controller/pkg/metrics/ovnkube_controller_test.go +++ b/go-controller/pkg/metrics/recorders/duration_test.go @@ -1,4 +1,4 @@ -package metrics +package recorders import ( "fmt" @@ -14,17 +14,29 @@ import ( "github.com/ovn-org/libovsdb/client" + egressfirewallfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/clientset/versioned/fake" + egressipfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/clientset/versioned/fake" + egressqosfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/clientset/versioned/fake" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics/mocks" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) -func setupOvn(nbData libovsdbtest.TestSetup) (client.Client, client.Client, *libovsdbtest.Context) { - nbClient, sbClient, cleanup, err := libovsdbtest.NewNBSBTestHarness(nbData) +func setHvCfg(nbClient client.Client, hvCfg int, hvCfgTimestamp time.Time) { + nbGlobal := nbdb.NBGlobal{} + nbGlobalResp, err := libovsdbops.GetNBGlobal(nbClient, &nbGlobal) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nbGlobalResp.HvCfg = hvCfg + nbGlobalResp.HvCfgTimestamp = int(hvCfgTimestamp.UnixMilli()) + ops, err := nbClient.Where(nbGlobalResp).Update(nbGlobalResp, &nbGlobalResp.HvCfg, &nbGlobalResp.HvCfgTimestamp) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ops).To(gomega.HaveLen(1)) + _, err = libovsdbops.TransactAndCheck(nbClient, ops) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return sbClient, nbClient, cleanup } func getKubeClient(nodeCount int) *kube.Kube { @@ -40,23 +52,16 @@ func getKubeClient(nodeCount int) *kube.Kube { return &kube.Kube{KClient: kubeFakeClient} } -func setHvCfg(nbClient client.Client, hvCfg int, hvCfgTimestamp time.Time) { - nbGlobal := nbdb.NBGlobal{} - nbGlobalResp, err := libovsdbops.GetNBGlobal(nbClient, &nbGlobal) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - nbGlobalResp.HvCfg = hvCfg - nbGlobalResp.HvCfgTimestamp = int(hvCfgTimestamp.UnixMilli()) - ops, err := nbClient.Where(nbGlobalResp).Update(nbGlobalResp, &nbGlobalResp.HvCfg, &nbGlobalResp.HvCfgTimestamp) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(ops).To(gomega.HaveLen(1)) - _, err = libovsdbops.TransactAndCheck(nbClient, ops) +func setupOvn(nbData libovsdbtest.TestSetup) (client.Client, client.Client, *libovsdbtest.Context) { + nbClient, sbClient, cleanup, err := libovsdbtest.NewNBSBTestHarness(nbData) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return sbClient, nbClient, cleanup } var _ = ginkgo.Describe("Config Duration Operations", func() { var ( instance *ConfigDurationRecorder - k *kube.Kube + wf *factory.WatchFactory nbClient client.Client cleanup *libovsdbtest.Context stop chan struct{} @@ -69,7 +74,23 @@ var _ = ginkgo.Describe("Config Duration Operations", func() { ginkgo.BeforeEach(func() { cdr = nil instance = GetConfigDurationRecorder() - k = getKubeClient(1) + k := getKubeClient(1) + egressFirewallFakeClient := &egressfirewallfake.Clientset{} + egressIPFakeClient := &egressipfake.Clientset{} + egressQoSFakeClient := &egressqosfake.Clientset{} + fakeClient := &util.OVNClientset{ + KubeClient: k.KClient, + EgressIPClient: egressIPFakeClient, + EgressFirewallClient: egressFirewallFakeClient, + EgressQoSClient: egressQoSFakeClient, + } + + var err error + wf, err = factory.NewMasterWatchFactory(fakeClient.GetMasterClientset()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = wf.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + stop = make(chan struct{}) _, nbClient, cleanup = setupOvn(libovsdbtest.TestSetup{ NBData: []libovsdbtest.TestData{&nbdb.NBGlobal{UUID: "cd-op-uuid"}}}) @@ -78,11 +99,12 @@ var _ = ginkgo.Describe("Config Duration Operations", func() { ginkgo.AfterEach(func() { cleanup.Cleanup() close(stop) + wf.Stop() }) ginkgo.Context("Runtime", func() { ginkgo.It("records correctly", func() { - instance.Run(nbClient, k, 0, time.Millisecond, stop) + instance.Run(nbClient, wf, 0, time.Millisecond, stop) histoMock := mocks.NewHistogramVecMock() metricNetworkProgramming = histoMock startTimestamp, ok := instance.Start("pod", testNamespaceA, testPodNameA) @@ -104,7 +126,7 @@ var _ = ginkgo.Describe("Config Duration Operations", func() { }) ginkgo.It("records correctly with OVN latency", func() { - instance.Run(nbClient, k, 0, time.Millisecond, stop) + instance.Run(nbClient, wf, 0, time.Millisecond, stop) histoMock := mocks.NewHistogramVecMock() metricNetworkProgramming = histoMock startTimestamp, ok := instance.Start("pod", testNamespaceA, testPodNameA) @@ -134,7 +156,7 @@ var _ = ginkgo.Describe("Config Duration Operations", func() { }) ginkgo.It("records multiple different objs including adding OVN latency", func() { - instance.Run(nbClient, k, 0, time.Millisecond, stop) + instance.Run(nbClient, wf, 0, time.Millisecond, stop) histoMock := mocks.NewHistogramVecMock() metricNetworkProgramming = histoMock // recording 1 @@ -186,13 +208,13 @@ var _ = ginkgo.Describe("Config Duration Operations", func() { }) ginkgo.It("denies recording when no start called", func() { - instance.Run(nbClient, k, 0, time.Millisecond, stop) + instance.Run(nbClient, wf, 0, time.Millisecond, stop) ops, _, _, _ := instance.AddOVN(nbClient, "pod", testNamespaceA, testPodNameA) gomega.Expect(ops).Should(gomega.BeEmpty()) }) ginkgo.It("allows multiple addOVN records for the same obj", func() { - instance.Run(nbClient, k, 0, time.Millisecond, stop) + instance.Run(nbClient, wf, 0, time.Millisecond, stop) histoMock := mocks.NewHistogramVecMock() metricNetworkProgramming = histoMock // recording 1 diff --git a/go-controller/pkg/node/default_node_network_controller.go b/go-controller/pkg/node/default_node_network_controller.go index 02a110b5d7..f9f3b36ec5 100644 --- a/go-controller/pkg/node/default_node_network_controller.go +++ b/go-controller/pkg/node/default_node_network_controller.go @@ -830,7 +830,7 @@ func (nc *DefaultNodeNetworkController) Init(ctx context.Context) error { } } - if node, err = nc.Kube.GetNode(nc.name); err != nil { + if node, err = nc.watchFactory.GetNode(nc.name); err != nil { return fmt.Errorf("error retrieving node %s: %v", nc.name, err) } @@ -895,7 +895,7 @@ func (nc *DefaultNodeNetworkController) Init(ctx context.Context) error { // First wait for the node logical switch to be created by the Master, timeout is 300s. err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 300*time.Second, true, func(_ context.Context) (bool, error) { - if node, err = nc.Kube.GetNode(nc.name); err != nil { + if node, err = nc.watchFactory.GetNode(nc.name); err != nil { klog.Infof("Waiting to retrieve node %s: %v", nc.name, err) return false, nil } @@ -999,7 +999,7 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error { klog.Errorf("Setting klog \"loglevel\" to 5 failed, err: %v", err) } - if node, err = nc.Kube.GetNode(nc.name); err != nil { + if node, err = nc.watchFactory.GetNode(nc.name); err != nil { return fmt.Errorf("error retrieving node %s: %v", nc.name, err) } @@ -1079,7 +1079,7 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error { err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 300*time.Second, true, func(_ context.Context) (bool, error) { // we loop through all the nodes in the cluster and ensure ovnkube-controller has finished creating the LRSR required for pod2pod overlay communication if !syncNodes { - nodes, err := nc.Kube.GetNodes() + nodes, err := nc.watchFactory.GetNodes() if err != nil { err1 = fmt.Errorf("upgrade hack: error retrieving node %s: %v", nc.name, err) return false, nil diff --git a/go-controller/pkg/node/gateway_egressip_test.go b/go-controller/pkg/node/gateway_egressip_test.go index bd09738200..db43f7450a 100644 --- a/go-controller/pkg/node/gateway_egressip_test.go +++ b/go-controller/pkg/node/gateway_egressip_test.go @@ -70,7 +70,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { isUpdated, err := addrMgr.addEgressIP(eip) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process a valid EgressIP") gomega.Expect(isUpdated).Should(gomega.BeTrue()) - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(ipV4Addr)) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -85,7 +85,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { isUpdated, err := addrMgr.addEgressIP(eip) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process a valid EgressIP") gomega.Expect(isUpdated).Should(gomega.BeFalse()) - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).ShouldNot(gomega.ConsistOf(ipV4Addr)) gomega.Expect(nlMock.AssertNotCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -100,7 +100,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { isUpdated, err := addrMgr.addEgressIP(eip) gomega.Expect(err).Should(gomega.HaveOccurred(), "should process a valid EgressIP") gomega.Expect(isUpdated).Should(gomega.BeFalse()) - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).ShouldNot(gomega.ConsistOf(ipV4Addr)) gomega.Expect(nlMock.AssertNotCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -120,7 +120,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { isUpdated, err := addrMgr.addEgressIP(eip) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process a valid EgressIP") gomega.Expect(isUpdated).Should(gomega.BeTrue()) - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(ipV4Addr, ipV4Addr2)) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -143,7 +143,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { isUpdated, err := addrMgr.updateEgressIP(unassignedEIP, assignedEIP) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process a valid EgressIP") gomega.Expect(isUpdated).Should(gomega.BeTrue()) - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(ipV4Addr)) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -168,7 +168,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { isUpdated, err = addrMgr.updateEgressIP(assignedEIP, unassignedEIP) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process a valid EgressIP") gomega.Expect(isUpdated).Should(gomega.BeTrue()) - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).ShouldNot(gomega.ConsistOf(ipV4Addr)) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -197,7 +197,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { isUpdated, err = addrMgr.updateEgressIP(assignedEIP1, assignedEIP2) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process a valid EgressIP") gomega.Expect(isUpdated).Should(gomega.BeTrue()) - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(ipV4Addr2)) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -227,7 +227,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { isUpdated, err = addrMgr.deleteEgressIP(eip) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process a valid EgressIP") gomega.Expect(isUpdated).Should(gomega.BeTrue()) - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).ShouldNot(gomega.ConsistOf(ipV4Addr)) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -243,7 +243,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { isUpdated, err := addrMgr.deleteEgressIP(eip) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process a valid EgressIP") gomega.Expect(isUpdated).Should(gomega.BeFalse()) - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(ipV4Addr2)) gomega.Expect(nlMock.AssertNotCalled(ginkgo.GinkgoT(), "AddrDel", nlLinkMock, @@ -267,7 +267,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { eipUnassigned3 := getEIPNotAssignedToNode(mark3, ipV4Addr3) err := addrMgr.syncEgressIP([]interface{}{eipAssigned1, eipAssigned2, eipUnassigned3}) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process valid EgressIPs") - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(ipV4Addr, ipV4Addr2)) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -291,7 +291,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { eipAssigned2 := getEIPAssignedToNode(nodeName, mark2, ipV4Addr2) err := addrMgr.syncEgressIP([]interface{}{eipAssigned1, eipAssigned2}) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process valid EgressIPs") - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.ConsistOf(ipV4Addr, ipV4Addr2)) gomega.Expect(nlMock.AssertCalled(ginkgo.GinkgoT(), "AddrAdd", nlLinkMock, @@ -308,7 +308,7 @@ var _ = ginkgo.Describe("Gateway EgressIP", func() { eipAssigned := getEIPAssignedToNode(nodeName, "", ipV4Addr) err := addrMgr.syncEgressIP([]interface{}{eipAssigned}) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "should process valid EgressIPs") - node, err := addrMgr.kube.GetNode(nodeName) + node, err := addrMgr.nodeLister.Get(nodeName) gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "node should be present within kapi") gomega.Expect(parseEIPsFromAnnotation(node)).Should(gomega.BeEmpty()) }) diff --git a/go-controller/pkg/node/gateway_udn_test.go b/go-controller/pkg/node/gateway_udn_test.go index 34673210ca..1227163480 100644 --- a/go-controller/pkg/node/gateway_udn_test.go +++ b/go-controller/pkg/node/gateway_udn_test.go @@ -459,7 +459,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { mgtPortMAC = util.IPAddrToHWAddr(util.GetNodeManagementIfAddr(ipNet).IP).String() getCreationFakeCommands(fexec, mgtPort, mgtPortMAC, netName, nodeName, netInfo.MTU()) nodeLister.On("Get", mock.AnythingOfType("string")).Return(node, nil) - factoryMock.On("GetNode", "worker1").Return(node, nil) + factoryMock.On("GetNodeForWindows", "worker1").Return(node, nil) err = testNS.Do(func(ns.NetNS) error { defer GinkgoRecover() @@ -502,7 +502,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { Expect(err).NotTo(HaveOccurred()) getDeletionFakeOVSCommands(fexec, mgtPort) nodeLister.On("Get", mock.AnythingOfType("string")).Return(node, nil) - factoryMock.On("GetNode", "worker1").Return(node, nil) + factoryMock.On("GetNodeForWindows", "worker1").Return(node, nil) cnode := node.DeepCopy() kubeMock.On("UpdateNodeStatus", cnode).Return(nil) // check if network key gets deleted from annotation err = testNS.Do(func(ns.NetNS) error { @@ -538,7 +538,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { mgtPortMAC = util.IPAddrToHWAddr(util.GetNodeManagementIfAddr(ipNet).IP).String() getCreationFakeCommands(fexec, mgtPort, mgtPortMAC, netName, nodeName, netInfo.MTU()) nodeLister.On("Get", mock.AnythingOfType("string")).Return(node, nil) - factoryMock.On("GetNode", "worker1").Return(node, nil) + factoryMock.On("GetNodeForWindows", "worker1").Return(node, nil) err = testNS.Do(func(ns.NetNS) error { defer GinkgoRecover() ofm := getDummyOpenflowManager() @@ -580,7 +580,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { Expect(err).NotTo(HaveOccurred()) getDeletionFakeOVSCommands(fexec, mgtPort) nodeLister.On("Get", mock.AnythingOfType("string")).Return(node, nil) - factoryMock.On("GetNode", "worker1").Return(node, nil) + factoryMock.On("GetNodeForWindows", "worker1").Return(node, nil) cnode := node.DeepCopy() kubeMock.On("UpdateNodeStatus", cnode).Return(nil) // check if network key gets deleted from annotation err = testNS.Do(func(ns.NetNS) error { diff --git a/go-controller/pkg/ovn/base_event_handler.go b/go-controller/pkg/ovn/base_event_handler.go index b08bc30b99..f54afc9925 100644 --- a/go-controller/pkg/ovn/base_event_handler.go +++ b/go-controller/pkg/ovn/base_event_handler.go @@ -14,7 +14,7 @@ import ( egressfirewall "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics/recorders" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -213,11 +213,11 @@ func (h *baseNetworkControllerEventHandler) recordAddEvent(objType reflect.Type, case factory.PolicyType: np := obj.(*knet.NetworkPolicy) klog.V(5).Infof("Recording add event on network policy %s/%s", np.Namespace, np.Name) - metrics.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) + recorders.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) case factory.MultiNetworkPolicyType: mnp := obj.(*mnpapi.MultiNetworkPolicy) klog.V(5).Infof("Recording add event on multinetwork policy %s/%s", mnp.Namespace, mnp.Name) - metrics.GetConfigDurationRecorder().Start("multinetworkpolicy", mnp.Namespace, mnp.Name) + recorders.GetConfigDurationRecorder().Start("multinetworkpolicy", mnp.Namespace, mnp.Name) } } @@ -227,11 +227,11 @@ func (h *baseNetworkControllerEventHandler) recordUpdateEvent(objType reflect.Ty case factory.PolicyType: np := obj.(*knet.NetworkPolicy) klog.V(5).Infof("Recording update event on network policy %s/%s", np.Namespace, np.Name) - metrics.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) + recorders.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) case factory.MultiNetworkPolicyType: mnp := obj.(*mnpapi.MultiNetworkPolicy) klog.V(5).Infof("Recording update event on multinetwork policy %s/%s", mnp.Namespace, mnp.Name) - metrics.GetConfigDurationRecorder().Start("multinetworkpolicy", mnp.Namespace, mnp.Name) + recorders.GetConfigDurationRecorder().Start("multinetworkpolicy", mnp.Namespace, mnp.Name) } } @@ -241,11 +241,11 @@ func (h *baseNetworkControllerEventHandler) recordDeleteEvent(objType reflect.Ty case factory.PolicyType: np := obj.(*knet.NetworkPolicy) klog.V(5).Infof("Recording delete event on network policy %s/%s", np.Namespace, np.Name) - metrics.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) + recorders.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) case factory.MultiNetworkPolicyType: mnp := obj.(*mnpapi.MultiNetworkPolicy) klog.V(5).Infof("Recording delete event on multinetwork policy %s/%s", mnp.Namespace, mnp.Name) - metrics.GetConfigDurationRecorder().Start("multinetworkpolicy", mnp.Namespace, mnp.Name) + recorders.GetConfigDurationRecorder().Start("multinetworkpolicy", mnp.Namespace, mnp.Name) } } @@ -255,10 +255,10 @@ func (h *baseNetworkControllerEventHandler) recordSuccessEvent(objType reflect.T case factory.PolicyType: np := obj.(*knet.NetworkPolicy) klog.V(5).Infof("Recording success event on network policy %s/%s", np.Namespace, np.Name) - metrics.GetConfigDurationRecorder().End("networkpolicy", np.Namespace, np.Name) + recorders.GetConfigDurationRecorder().End("networkpolicy", np.Namespace, np.Name) case factory.MultiNetworkPolicyType: mnp := obj.(*mnpapi.MultiNetworkPolicy) klog.V(5).Infof("Recording success event on multinetwork policy %s/%s", mnp.Namespace, mnp.Name) - metrics.GetConfigDurationRecorder().End("multinetworkpolicy", mnp.Namespace, mnp.Name) + recorders.GetConfigDurationRecorder().End("multinetworkpolicy", mnp.Namespace, mnp.Name) } } diff --git a/go-controller/pkg/ovn/base_network_controller.go b/go-controller/pkg/ovn/base_network_controller.go index 1a3f8685e4..db56f42cb9 100644 --- a/go-controller/pkg/ovn/base_network_controller.go +++ b/go-controller/pkg/ovn/base_network_controller.go @@ -14,8 +14,6 @@ import ( corev1 "k8s.io/api/core/v1" knet "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -35,6 +33,7 @@ import ( libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics/recorders" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/observability" @@ -241,7 +240,7 @@ func (oc *BaseNetworkController) reconcile(netInfo util.NetInfo, setNodeFailed f } if reconcilePendingPods { - if err := ovnretry.RequeuePendingPods(oc.kube, oc.GetNetInfo(), oc.retryPods); err != nil { + if err := ovnretry.RequeuePendingPods(oc.watchFactory, oc.GetNetInfo(), oc.retryPods); err != nil { klog.Errorf("Failed to requeue pending pods for network %s: %v", oc.GetNetworkName(), err) } } @@ -319,7 +318,7 @@ func (bnc *BaseNetworkController) GetLogicalPortName(pod *corev1.Pod, nadName st func (bnc *BaseNetworkController) AddConfigDurationRecord(kind, namespace, name string) ( []ovsdb.Operation, func(), time.Time, error) { if !bnc.IsSecondary() { - return metrics.GetConfigDurationRecorder().AddOVN(bnc.nbClient, kind, namespace, name) + return recorders.GetConfigDurationRecorder().AddOVN(bnc.nbClient, kind, namespace, name) } // TBD: no op for secondary network for now return []ovsdb.Operation{}, func() {}, time.Time{}, nil @@ -579,12 +578,10 @@ func (bnc *BaseNetworkController) deleteNodeLogicalNetwork(nodeName string) erro func (bnc *BaseNetworkController) addAllPodsOnNode(nodeName string) []error { errs := []error{} - pods, err := bnc.kube.GetPods(metav1.NamespaceAll, metav1.ListOptions{ - FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String(), - }) + pods, err := bnc.watchFactory.GetAllPods() if err != nil { errs = append(errs, err) - klog.Errorf("Unable to list existing pods on node: %s, existing pods on this node may not function", + klog.Errorf("Unable to list existing pods for synchronizing node: %s, existing pods on this node may not function", nodeName) } else { klog.V(5).Infof("When adding node %s for network %s, found %d pods to add to retryPods", nodeName, bnc.GetNetworkName(), len(pods)) @@ -593,6 +590,9 @@ func (bnc *BaseNetworkController) addAllPodsOnNode(nodeName string) []error { if util.PodCompleted(&pod) { continue } + if pod.Spec.NodeName != nodeName { + continue + } klog.V(5).Infof("Adding pod %s/%s to retryPods for network %s", pod.Namespace, pod.Name, bnc.GetNetworkName()) err = bnc.retryPods.AddRetryObjWithAddNoBackoff(&pod) if err != nil { diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/metrics.go b/go-controller/pkg/ovn/controller/admin_network_policy/metrics.go index cd2e636ea8..dcf1fb6aab 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/metrics.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/metrics.go @@ -8,14 +8,14 @@ import ( anpapi "sigs.k8s.io/network-policy-api/apis/v1alpha1" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" ) // Descriptors used by the ANPControllerCollector below. var ( anpRuleCountDesc = prometheus.NewDesc( - prometheus.BuildFQName(metrics.MetricOvnkubeNamespace, metrics.MetricOvnkubeSubsystemController, "admin_network_policies_rules"), + prometheus.BuildFQName(types.MetricOvnkubeNamespace, types.MetricOvnkubeSubsystemController, "admin_network_policies_rules"), "The total number of rules across all admin network policies in the cluster", []string{ "direction", // direction is either "ingress" or "egress"; so cardinality is max 2 for this label @@ -23,7 +23,7 @@ var ( }, nil, ) banpRuleCountDesc = prometheus.NewDesc( - prometheus.BuildFQName(metrics.MetricOvnkubeNamespace, metrics.MetricOvnkubeSubsystemController, "baseline_admin_network_policies_rules"), + prometheus.BuildFQName(types.MetricOvnkubeNamespace, types.MetricOvnkubeSubsystemController, "baseline_admin_network_policies_rules"), "The total number of rules across all baseline admin network policies in the cluster", []string{ "direction", // direction is either "ingress" or "egress"; so cardinality is max 2 for this label diff --git a/go-controller/pkg/ovn/controller/network_qos/metrics.go b/go-controller/pkg/ovn/controller/network_qos/metrics.go index 96fa30834d..de5c6872ca 100644 --- a/go-controller/pkg/ovn/controller/network_qos/metrics.go +++ b/go-controller/pkg/ovn/controller/network_qos/metrics.go @@ -3,15 +3,15 @@ package networkqos import ( "github.com/prometheus/client_golang/prometheus" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" ) // Metrics to be exposed var ( nqosCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: metrics.MetricOvnkubeNamespace, - Subsystem: metrics.MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "num_network_qoses", Help: "The total number of network qoses in the cluster", }, @@ -20,8 +20,8 @@ var ( nqosOvnOperationDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: metrics.MetricOvnkubeNamespace, - Subsystem: metrics.MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "nqos_ovn_operation_duration_ms", Help: "Time spent on reconciling a NetworkQoS event", Buckets: prometheus.ExponentialBuckets(.1, 2, 15), @@ -31,8 +31,8 @@ var ( nqosReconcileDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: metrics.MetricOvnkubeNamespace, - Subsystem: metrics.MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "nqos_creation_duration_ms", Help: "Time spent on reconciling a NetworkQoS event", Buckets: prometheus.ExponentialBuckets(.1, 2, 15), @@ -42,8 +42,8 @@ var ( nqosPodReconcileDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: metrics.MetricOvnkubeNamespace, - Subsystem: metrics.MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "nqos_deletion_duration_ms", Help: "Time spent on reconciling a Pod event", Buckets: prometheus.ExponentialBuckets(.1, 2, 15), @@ -53,8 +53,8 @@ var ( nqosNamespaceReconcileDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: metrics.MetricOvnkubeNamespace, - Subsystem: metrics.MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "nqos_ns_reconcile_duration_ms", Help: "Time spent on reconciling Namespace change for all Pods related to NetworkQoSes", Buckets: prometheus.ExponentialBuckets(.1, 2, 15), @@ -64,8 +64,8 @@ var ( nqosStatusPatchDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: metrics.MetricOvnkubeNamespace, - Subsystem: metrics.MetricOvnkubeSubsystemController, + Namespace: types.MetricOvnkubeNamespace, + Subsystem: types.MetricOvnkubeSubsystemController, Name: "nqos_status_patch_duration_ms", Help: "Time spent on patching the status of a NetworkQoS", }, diff --git a/go-controller/pkg/ovn/controller/services/loadbalancer.go b/go-controller/pkg/ovn/controller/services/loadbalancer.go index ba4eebc43a..8c3d1c9114 100644 --- a/go-controller/pkg/ovn/controller/services/loadbalancer.go +++ b/go-controller/pkg/ovn/controller/services/loadbalancer.go @@ -14,7 +14,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics/recorders" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -283,7 +283,7 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing return err } - recordOps, txOkCallBack, _, err := metrics.GetConfigDurationRecorder().AddOVN(nbClient, "service", + recordOps, txOkCallBack, _, err := recorders.GetConfigDurationRecorder().AddOVN(nbClient, "service", service.Namespace, service.Name) if err != nil { klog.Errorf("Failed to record config duration: %v", err) diff --git a/go-controller/pkg/ovn/controller/services/services_controller.go b/go-controller/pkg/ovn/controller/services/services_controller.go index 031a38a0d6..e03ad40b5c 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller.go +++ b/go-controller/pkg/ovn/controller/services/services_controller.go @@ -36,6 +36,7 @@ import ( libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics/recorders" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -282,7 +283,7 @@ func (c *Controller) handleErr(err error, key string) { klog.ErrorS(err, "Failed to split meta namespace cache key", "key", key) } if err == nil { - metrics.GetConfigDurationRecorder().End("service", ns, name) + recorders.GetConfigDurationRecorder().End("service", ns, name) c.queue.Forget(key) return } @@ -296,7 +297,7 @@ func (c *Controller) handleErr(err error, key string) { } klog.Warningf("Dropping service %q out of the queue for network=%s: %v", key, c.netInfo.GetNetworkName(), err) - metrics.GetConfigDurationRecorder().End("service", ns, name) + recorders.GetConfigDurationRecorder().End("service", ns, name) c.queue.Forget(key) utilruntime.HandleError(err) } @@ -609,7 +610,7 @@ func (c *Controller) onServiceAdd(obj interface{}) { if c.skipService(service.Name, service.Namespace) { return } - metrics.GetConfigDurationRecorder().Start("service", service.Namespace, service.Name) + recorders.GetConfigDurationRecorder().Start("service", service.Namespace, service.Name) klog.V(5).Infof("Adding service %s for network=%s", key, c.netInfo.GetNetworkName()) c.queue.Add(key) } @@ -631,7 +632,7 @@ func (c *Controller) onServiceUpdate(oldObj, newObj interface{}) { return } - metrics.GetConfigDurationRecorder().Start("service", newService.Namespace, newService.Name) + recorders.GetConfigDurationRecorder().Start("service", newService.Namespace, newService.Name) c.queue.Add(key) } } @@ -651,7 +652,7 @@ func (c *Controller) onServiceDelete(obj interface{}) { klog.V(4).Infof("Deleting service %s for network=%s", key, c.netInfo.GetNetworkName()) - metrics.GetConfigDurationRecorder().Start("service", service.Namespace, service.Name) + recorders.GetConfigDurationRecorder().Start("service", service.Namespace, service.Name) c.queue.Add(key) } diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index cf6886e846..3ea00bcb17 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -22,6 +22,7 @@ import ( egressqoslisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/listers/egressqos/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics/recorders" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/observability" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" @@ -356,7 +357,7 @@ func (oc *DefaultNetworkController) Stop() { // // If true, then either quit or perform a complete reconfiguration of the cluster (recreate switches/routers with new subnet values) func (oc *DefaultNetworkController) init() error { - existingNodes, err := oc.kube.GetNodes() + existingNodes, err := oc.watchFactory.GetNodes() if err != nil { klog.Errorf("Error in fetching nodes: %v", err) return err @@ -657,11 +658,11 @@ func (h *defaultNetworkControllerEventHandler) RecordAddEvent(obj interface{}) { pod := obj.(*corev1.Pod) klog.V(5).Infof("Recording add event on pod %s/%s", pod.Namespace, pod.Name) h.oc.podRecorder.AddPod(pod.UID) - metrics.GetConfigDurationRecorder().Start("pod", pod.Namespace, pod.Name) + recorders.GetConfigDurationRecorder().Start("pod", pod.Namespace, pod.Name) case factory.PolicyType: np := obj.(*knet.NetworkPolicy) klog.V(5).Infof("Recording add event on network policy %s/%s", np.Namespace, np.Name) - metrics.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) + recorders.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) } } @@ -671,11 +672,11 @@ func (h *defaultNetworkControllerEventHandler) RecordUpdateEvent(obj interface{} case factory.PodType: pod := obj.(*corev1.Pod) klog.V(5).Infof("Recording update event on pod %s/%s", pod.Namespace, pod.Name) - metrics.GetConfigDurationRecorder().Start("pod", pod.Namespace, pod.Name) + recorders.GetConfigDurationRecorder().Start("pod", pod.Namespace, pod.Name) case factory.PolicyType: np := obj.(*knet.NetworkPolicy) klog.V(5).Infof("Recording update event on network policy %s/%s", np.Namespace, np.Name) - metrics.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) + recorders.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) } } @@ -686,11 +687,11 @@ func (h *defaultNetworkControllerEventHandler) RecordDeleteEvent(obj interface{} pod := obj.(*corev1.Pod) klog.V(5).Infof("Recording delete event on pod %s/%s", pod.Namespace, pod.Name) h.oc.podRecorder.CleanPod(pod.UID) - metrics.GetConfigDurationRecorder().Start("pod", pod.Namespace, pod.Name) + recorders.GetConfigDurationRecorder().Start("pod", pod.Namespace, pod.Name) case factory.PolicyType: np := obj.(*knet.NetworkPolicy) klog.V(5).Infof("Recording delete event on network policy %s/%s", np.Namespace, np.Name) - metrics.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) + recorders.GetConfigDurationRecorder().Start("networkpolicy", np.Namespace, np.Name) } } @@ -700,11 +701,11 @@ func (h *defaultNetworkControllerEventHandler) RecordSuccessEvent(obj interface{ case factory.PodType: pod := obj.(*corev1.Pod) klog.V(5).Infof("Recording success event on pod %s/%s", pod.Namespace, pod.Name) - metrics.GetConfigDurationRecorder().End("pod", pod.Namespace, pod.Name) + recorders.GetConfigDurationRecorder().End("pod", pod.Namespace, pod.Name) case factory.PolicyType: np := obj.(*knet.NetworkPolicy) klog.V(5).Infof("Recording success event on network policy %s/%s", np.Namespace, np.Name) - metrics.GetConfigDurationRecorder().End("networkpolicy", np.Namespace, np.Name) + recorders.GetConfigDurationRecorder().End("networkpolicy", np.Namespace, np.Name) } } diff --git a/go-controller/pkg/ovn/gateway.go b/go-controller/pkg/ovn/gateway.go index 9ecbb512fd..7c38289737 100644 --- a/go-controller/pkg/ovn/gateway.go +++ b/go-controller/pkg/ovn/gateway.go @@ -11,8 +11,6 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" utilnet "k8s.io/utils/net" @@ -161,9 +159,7 @@ func (gw *GatewayManager) cleanupStalePodSNATs(nodeName string, nodeIPs []*net.I // the SNATs stale podIPsWithSNAT := sets.New[string]() if !gw.isRoutingAdvertised(nodeName) && config.Gateway.DisableSNATMultipleGWs { - pods, err := gw.kube.GetPods(metav1.NamespaceAll, metav1.ListOptions{ - FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String(), - }) + pods, err := gw.watchFactory.GetAllPods() if err != nil { return fmt.Errorf("unable to list existing pods on node: %s, %w", nodeName, err) @@ -173,6 +169,9 @@ func (gw *GatewayManager) cleanupStalePodSNATs(nodeName string, nodeIPs []*net.I if !util.PodScheduled(&pod) { //if the pod is not scheduled we should not remove the nat continue } + if pod.Spec.NodeName != nodeName { + continue + } if util.PodCompleted(&pod) { collidingPod, err := findPodWithIPAddresses(gw.watchFactory, gw.netInfo, []net.IP{utilnet.ParseIPSloppy(pod.Status.PodIP)}, "") //even if a pod is completed we should still delete the nat if the ip is not in use anymore if err != nil { diff --git a/go-controller/pkg/ovn/gateway_test.go b/go-controller/pkg/ovn/gateway_test.go index da48869991..57f5fb4be2 100644 --- a/go-controller/pkg/ovn/gateway_test.go +++ b/go-controller/pkg/ovn/gateway_test.go @@ -220,13 +220,17 @@ func generateGatewayInitExpectedNB(testData []libovsdbtest.TestData, expectedOVN natUUID := fmt.Sprintf("nat-%d-UUID", i) natUUIDs = append(natUUIDs, natUUID) physicalIP, _ := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(subnet), l3GatewayConfig.IPAddresses) - testData = append(testData, &nbdb.NAT{ + nat := nbdb.NAT{ UUID: natUUID, ExternalIP: physicalIP.IP.String(), LogicalIP: subnet.String(), Options: map[string]string{"stateless": "false"}, Type: nbdb.NATTypeSNAT, - }) + } + if config.Gateway.Mode != config.GatewayModeDisabled { + nat.ExternalPortRange = config.DefaultEphemeralPortRange + } + testData = append(testData, &nat) } } @@ -234,13 +238,17 @@ func generateGatewayInitExpectedNB(testData []libovsdbtest.TestData, expectedOVN natUUID := fmt.Sprintf("nat-join-%d-UUID", i) natUUIDs = append(natUUIDs, natUUID) joinLRPIP, _ := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(physicalIP), joinLRPIPs) - testData = append(testData, &nbdb.NAT{ + nat := nbdb.NAT{ UUID: natUUID, ExternalIP: physicalIP.IP.String(), LogicalIP: joinLRPIP.IP.String(), Options: map[string]string{"stateless": "false"}, Type: nbdb.NATTypeSNAT, - }) + } + if config.Gateway.Mode != config.GatewayModeDisabled { + nat.ExternalPortRange = config.DefaultEphemeralPortRange + } + testData = append(testData, &nat) } testData = append(testData, &nbdb.MeterBand{ @@ -394,6 +402,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ginkgo.Context("Gateway Creation Operations Shared Gateway Mode", func() { ginkgo.BeforeEach(func() { config.Gateway.Mode = config.GatewayModeShared + config.Gateway.EphemeralPortRange = config.DefaultEphemeralPortRange }) ginkgo.It("creates an IPv4 gateway in OVN", func() { @@ -1441,6 +1450,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ginkgo.BeforeEach(func() { config.Gateway.Mode = config.GatewayModeLocal config.IPv6Mode = false + config.Gateway.EphemeralPortRange = config.DefaultEphemeralPortRange }) ginkgo.It("creates a dual-stack gateway in OVN", func() { diff --git a/go-controller/pkg/ovn/hybrid.go b/go-controller/pkg/ovn/hybrid.go index f7debe4ec8..6386d719e9 100644 --- a/go-controller/pkg/ovn/hybrid.go +++ b/go-controller/pkg/ovn/hybrid.go @@ -171,7 +171,7 @@ func (oc *DefaultNetworkController) setupHybridLRPolicySharedGw(nodeSubnets []*n // In cases of OpenShift SDN live migration, where config.HybridOverlay.ClusterSubnets is not provided, we // use the host subnets allocated by OpenShiftSDN as the hybrid-overlay-node-subnet and set up hybrid // overlay routes/policies to these subnets. - nodes, err := oc.kube.GetNodes() + nodes, err := oc.watchFactory.GetNodes() if err != nil { return err } @@ -407,7 +407,7 @@ func (oc *DefaultNetworkController) removeRoutesToHONodeSubnet(nodeName string, } // Delete routes to HO subnet from GRs - nodes, err := oc.kube.GetNodes() + nodes, err := oc.watchFactory.GetNodes() if err != nil { return err } diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 26eb1277fe..78b5ce2402 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -730,7 +730,7 @@ func (oc *DefaultNetworkController) addUpdateHoNodeEvent(node *corev1.Node) erro return err } - nodes, err := oc.kube.GetNodes() + nodes, err := oc.watchFactory.GetNodes() if err != nil { return err } diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go index 8d4b57dda7..330ffeaddf 100644 --- a/go-controller/pkg/ovn/master_test.go +++ b/go-controller/pkg/ovn/master_test.go @@ -1167,7 +1167,7 @@ var _ = ginkgo.Describe("Default network controller operations", func() { types.OVNClusterRouter, badRoute, p) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Syncing node with OVNK") - node, err := oc.kube.GetNode(testNode.Name) + node, err := oc.kube.GetNodeForWindows(testNode.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = oc.syncNodeManagementPortDefault(node, node.Name, []*net.IPNet{subnet}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/go-controller/pkg/ovn/namespace_test.go b/go-controller/pkg/ovn/namespace_test.go index c067098709..3e8c556b8a 100644 --- a/go-controller/pkg/ovn/namespace_test.go +++ b/go-controller/pkg/ovn/namespace_test.go @@ -238,6 +238,7 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { ginkgo.It("creates an address set for existing nodes when the host network traffic namespace is created", func() { config.Gateway.Mode = config.GatewayModeShared config.Gateway.NodeportEnable = true + config.Gateway.EphemeralPortRange = config.DefaultEphemeralPortRange var err error config.Default.ClusterSubnets, err = config.ParseClusterSubnetEntries(clusterCIDR) gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/go-controller/pkg/ovn/ovn_test.go b/go-controller/pkg/ovn/ovn_test.go index 0f2a9f1058..801777854a 100644 --- a/go-controller/pkg/ovn/ovn_test.go +++ b/go-controller/pkg/ovn/ovn_test.go @@ -276,7 +276,7 @@ func (o *FakeOVN) init(nadList []nettypes.NetworkAttachmentDefinition) { err = o.eIPController.SyncLocalNodeZonesCache() gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "syncing Nodes OVN zones status must succeed to support EgressIP") - existingNodes, err := o.controller.kube.GetNodes() + existingNodes, err := o.controller.watchFactory.GetNodes() if err == nil { for _, node := range existingNodes { o.controller.localZoneNodes.Store(node.Name, true) diff --git a/go-controller/pkg/ovn/secondary_localnet_network_controller.go b/go-controller/pkg/ovn/secondary_localnet_network_controller.go index 3c6fef1027..4046f819ce 100644 --- a/go-controller/pkg/ovn/secondary_localnet_network_controller.go +++ b/go-controller/pkg/ovn/secondary_localnet_network_controller.go @@ -15,7 +15,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/pod" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics/recorders" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" @@ -65,7 +65,7 @@ func (h *secondaryLocalnetNetworkControllerEventHandler) RecordAddEvent(obj inte case factory.MultiNetworkPolicyType: mnp := obj.(*mnpapi.MultiNetworkPolicy) klog.V(5).Infof("Recording add event on multinetwork policy %s/%s", mnp.Namespace, mnp.Name) - metrics.GetConfigDurationRecorder().Start("multinetworkpolicy", mnp.Namespace, mnp.Name) + recorders.GetConfigDurationRecorder().Start("multinetworkpolicy", mnp.Namespace, mnp.Name) } } diff --git a/go-controller/pkg/ovndbmanager/ovndbmanager.go b/go-controller/pkg/ovndbmanager/ovndbmanager.go index 2c5fac6ab9..7e26bb2b98 100644 --- a/go-controller/pkg/ovndbmanager/ovndbmanager.go +++ b/go-controller/pkg/ovndbmanager/ovndbmanager.go @@ -226,7 +226,7 @@ func ensureClusterRaftMembership(db *util.OvsDbProperties, kclient kube.Interfac r = regexp.MustCompile(`([a-z0-9]{4}) at ` + dbServerRegexp) members := r.FindAllStringSubmatch(out, -1) kickedMembersCount := 0 - dbPods, err := kclient.GetPods(config.Kubernetes.OVNConfigNamespace, metav1.ListOptions{ + dbPods, err := kclient.GetPodsForDBChecker(config.Kubernetes.OVNConfigNamespace, metav1.ListOptions{ LabelSelector: labels.Set(map[string]string{"ovn-db-pod": "true"}).String(), }) if err != nil { diff --git a/go-controller/pkg/retry/obj_retry.go b/go-controller/pkg/retry/obj_retry.go index 46484ae4fe..fc17d07b50 100644 --- a/go-controller/pkg/retry/obj_retry.go +++ b/go-controller/pkg/retry/obj_retry.go @@ -9,14 +9,11 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/syncmap" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -772,38 +769,35 @@ func (r *RetryFramework) WatchResourceFiltered(namespaceForFilteredHandler strin return handler, nil } -// getPendingPods returns all pods that are in the Pending state -func getPendingPods(kubeClient kube.InterfaceOVN) ([]*corev1.Pod, error) { - var allPods []*corev1.Pod - - pods, err := kubeClient.GetPods(corev1.NamespaceAll, metav1.ListOptions{ - FieldSelector: fields.OneTermEqualSelector("status.phase", string(corev1.PodPending)).String(), - }) - if err != nil { - return nil, err - } - allPods = append(allPods, pods...) - return allPods, nil -} - // RequeuePendingPods enqueues all Pending pods into the retryPods associated with netInfo. -func RequeuePendingPods(kubeClient kube.InterfaceOVN, netInfo util.NetInfo, retryPods *RetryFramework) error { +func RequeuePendingPods(wf *factory.WatchFactory, netInfo util.NetInfo, retryPods *RetryFramework) error { var errs []error // NOTE: A pod may reference a NAD from a different namespace, so check all pending pods. - allPods, err := getPendingPods(kubeClient) + allPods, err := wf.GetAllPods() if err != nil { - return err + return fmt.Errorf("failed to get all pods: %w", err) } + podsAdded := false for _, pod := range allPods { pod := *pod + if !util.PodScheduled(&pod) { + continue + } + if pod.Status.Phase != corev1.PodPending { + continue + } klog.V(5).Infof("Adding pending pod %s/%s to retryPods for network %s", pod.Namespace, pod.Name, netInfo.GetNetworkName()) err := retryPods.AddRetryObjWithAddNoBackoff(&pod) if err != nil { errs = append(errs, err) + continue } + podsAdded = true + } + if podsAdded { + retryPods.RequestRetryObjs() } - retryPods.RequestRetryObjs() return utilerrors.Join(errs...) } diff --git a/go-controller/pkg/types/const.go b/go-controller/pkg/types/const.go index b06e337efd..92e5fc54c1 100644 --- a/go-controller/pkg/types/const.go +++ b/go-controller/pkg/types/const.go @@ -317,4 +317,17 @@ const ( // NFTNoPMTUDRemoteNodeIPsv6 is a set used to track remote node IPs that do not belong to // the local node's subnet. NFTNoPMTUDRemoteNodeIPsv6 = "no-pmtud-remote-node-ips-v6" + + // Metrics + MetricOvnkubeNamespace = "ovnkube" + MetricOvnkubeSubsystemController = "controller" + MetricOvnkubeSubsystemClusterManager = "clustermanager" + MetricOvnkubeSubsystemNode = "node" + MetricOvnNamespace = "ovn" + MetricOvnSubsystemDB = "db" + MetricOvnSubsystemNorthd = "northd" + MetricOvnSubsystemController = "controller" + MetricOvsNamespace = "ovs" + MetricOvsSubsystemVswitchd = "vswitchd" + MetricOvsSubsystemDB = "db" )