diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad404d04c8..5b3d11a7a4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -462,7 +462,7 @@ jobs: - {"target": "control-plane", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "control-plane", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "2br", "ic": "ic-single-node-zones"} - {"target": "control-plane", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "2br", "ic": "ic-single-node-zones"} - - {"target": "multi-homing", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-disabled"} + - {"target": "multi-homing", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "multi-homing-helm", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-disabled", "network-segmentation": "enable-network-segmentation"} - {"target": "node-ip-mac-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-disabled"} - {"target": "node-ip-mac-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} diff --git a/go-controller/pkg/node/bridgeconfig/bridgeflows.go b/go-controller/pkg/node/bridgeconfig/bridgeflows.go index 8a858c30e9..2fd111cfac 100644 --- a/go-controller/pkg/node/bridgeconfig/bridgeflows.go +++ b/go-controller/pkg/node/bridgeconfig/bridgeflows.go @@ -632,12 +632,12 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e } if ofPortPhys != "" { - // table 0, packets coming from external or other localnet ports. Send it through conntrack and - // resubmit to table 1 to know the state and mark of the connection. + // table 0, packets coming from external or other localnet ports and destined to OVN or LOCAL. + // Send it through conntrack and resubmit to table 1 to know the state and mark of the connection. // Note, there are higher priority rules that take care of traffic coming from LOCAL and OVN ports. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=50, ip, actions=ct(zone=%d, nat, table=1)", - nodetypes.DefaultOpenFlowCookie, config.Default.ConntrackZone)) + fmt.Sprintf("cookie=%s, priority=50, ip, dl_dst=%s, actions=ct(zone=%d, nat, table=1)", + nodetypes.DefaultOpenFlowCookie, bridgeMacAddress, config.Default.ConntrackZone)) } } @@ -733,8 +733,8 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e // table 0, packets coming from external. Send it through conntrack and // resubmit to table 1 to know the state and mark of the connection. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=50, in_port=%s, ipv6, "+ - "actions=ct(zone=%d, nat, table=1)", nodetypes.DefaultOpenFlowCookie, ofPortPhys, config.Default.ConntrackZone)) + fmt.Sprintf("cookie=%s, priority=50, ipv6, dl_dst=%s, actions=ct(zone=%d, nat, table=1)", + nodetypes.DefaultOpenFlowCookie, bridgeMacAddress, config.Default.ConntrackZone)) } } if ofPortPhys != "" { diff --git a/test/e2e/multihoming.go b/test/e2e/multihoming.go index 66d56e363a..bb82124927 100644 --- a/test/e2e/multihoming.go +++ b/test/e2e/multihoming.go @@ -4,11 +4,9 @@ import ( "context" "errors" "fmt" - "net/netip" "strings" "time" - "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/ovn-org/ovn-kubernetes/test/e2e/feature" @@ -25,8 +23,6 @@ import ( nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" nadclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" - ipgenerator "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/generator/ip" - util "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "github.com/ovn-org/ovn-kubernetes/test/e2e/deploymentconfig" "github.com/ovn-org/ovn-kubernetes/test/e2e/images" "github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider" @@ -36,6 +32,12 @@ import ( const ( PolicyForAnnotation = "k8s.v1.cni.cncf.io/policy-for" nodeHostnameKey = "kubernetes.io/hostname" + + externalNetworkSubnetV4 = "172.20.0.0/16" + externalNetworkSubnetV6 = "fd00:20::/64" + + fromHostSubnet = "from-host-subnet" // the test will generate an IP from the host subnet + fromExternalNetwork = "from-external-network" // the test will generate an IP from a subnet that the cluster is not aware of ) var _ = Describe("Multi Homing", feature.MultiHoming, func() { @@ -73,7 +75,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { }) Context("A single pod with an OVN-K secondary network", func() { - ginkgo.DescribeTable("is able to get to the Running phase", func(netConfigParams networkAttachmentConfigParams, podConfig podConfiguration) { + DescribeTable("is able to get to the Running phase", func(netConfigParams networkAttachmentConfigParams, podConfig podConfiguration) { netConfig := newNetworkAttachmentConfig(netConfigParams) netConfig.namespace = f.Namespace.Name @@ -124,7 +126,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { } } }, - ginkgo.Entry( + Entry( "when attaching to an L3 - routed - network", networkAttachmentConfigParams{ cidr: netCIDR(secondaryNetworkCIDR, netPrefixLengthPerNode), @@ -136,7 +138,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to an L3 - routed - network with IPv6 network", networkAttachmentConfigParams{ cidr: netCIDR(secondaryIPv6CIDR, netPrefixLengthIPv6PerNode), @@ -148,7 +150,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to an L2 - switched - network", networkAttachmentConfigParams{ cidr: secondaryFlatL2NetworkCIDR, @@ -160,7 +162,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to an L2 - switched - network featuring `excludeCIDR`s", networkAttachmentConfigParams{ cidr: secondaryFlatL2NetworkCIDR, @@ -173,7 +175,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to an L2 - switched - network without IPAM", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -184,7 +186,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to an L2 - switched - network with an IPv6 subnet", networkAttachmentConfigParams{ cidr: secondaryIPv6CIDR, @@ -196,7 +198,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to an L2 - switched - network with a dual stack configuration", networkAttachmentConfigParams{ cidr: strings.Join([]string{secondaryFlatL2NetworkCIDR, secondaryIPv6CIDR}, ","), @@ -208,7 +210,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to a localnet - switched - network", networkAttachmentConfigParams{ cidr: secondaryLocalnetNetworkCIDR, @@ -221,7 +223,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to a localnet - switched - network featuring `excludeCIDR`s", networkAttachmentConfigParams{ cidr: secondaryLocalnetNetworkCIDR, @@ -235,7 +237,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to a localnet - switched - network without IPAM", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -247,7 +249,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to a localnet - switched - network with an IPv6 subnet", networkAttachmentConfigParams{ cidr: secondaryIPv6CIDR, @@ -260,7 +262,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { name: podName, }, ), - ginkgo.Entry( + Entry( "when attaching to an L2 - switched - network with a dual stack configuration", networkAttachmentConfigParams{ cidr: strings.Join([]string{secondaryLocalnetNetworkCIDR, secondaryIPv6CIDR}, ","), @@ -276,19 +278,19 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { ) const ( - clientPodName = "client-pod" - clientIPOffset = 100 - serverIPOffset = 102 - port = 9000 + clientPodName = "client-pod" + clientIPOffset = 100 // offset for IP generation from a given subnet for client pod + serverIPOffset = 102 + externalRouterIPOffset = 55 + port = 9000 ) - ginkgo.DescribeTable("attached to a localnet network mapped to external primary interface bridge", //nolint:lll - + DescribeTable("attached to a localnet network mapped to external primary interface bridge", //nolint:lll func(netConfigParams networkAttachmentConfigParams, clientPodConfig, serverPodConfig podConfiguration, isCollocatedPods bool) { - By("Get two scheduable nodes and ensure client and server are located on distinct Nodes") + By("Get two schedulable nodes and ensure client and server are located on distinct Nodes") nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.Background(), f.ClientSet, 2) - framework.ExpectNoError(err, "2 scheduable nodes are required") - Expect(len(nodes.Items)).To(BeNumerically(">=", 1), "cluster should have at least 2 nodes") + framework.ExpectNoError(err, "2 schedulable nodes are required") + Expect(len(nodes.Items)).To(BeNumerically(">", 1), "cluster should have at least 2 nodes") if isCollocatedPods { clientPodConfig.nodeSelector = map[string]string{nodeHostnameKey: nodes.Items[0].GetName()} serverPodConfig.nodeSelector = map[string]string{nodeHostnameKey: nodes.Items[0].GetName()} @@ -296,11 +298,9 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { clientPodConfig.nodeSelector = map[string]string{nodeHostnameKey: nodes.Items[0].GetName()} serverPodConfig.nodeSelector = map[string]string{nodeHostnameKey: nodes.Items[1].GetName()} } - netConfig := newNetworkAttachmentConfig(networkAttachmentConfigParams{ - name: secondaryNetworkName, - namespace: f.Namespace.Name, - topology: "localnet", - }) + + netConfigParams.namespace = f.Namespace.Name + netConfig := newNetworkAttachmentConfig(netConfigParams) if clientPodConfig.namespace == "" { clientPodConfig.namespace = f.Namespace.Name } @@ -323,13 +323,13 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { ) Expect(err).NotTo(HaveOccurred()) - if serverPodConfig.attachments != nil && serverPodConfig.needsIPRequestFromHostSubnet { + if len(serverPodConfig.attachments) > 0 && serverPodConfig.ipRequestFromSubnet != "" { By("finalizing the server pod IP configuration") err = addIPRequestToPodConfig(cs, &serverPodConfig, serverIPOffset) Expect(err).NotTo(HaveOccurred()) } - if clientPodConfig.attachments != nil && clientPodConfig.needsIPRequestFromHostSubnet { + if len(clientPodConfig.attachments) > 0 && clientPodConfig.ipRequestFromSubnet != "" { By("finalizing the client pod IP configuration") err = addIPRequestToPodConfig(cs, &clientPodConfig, clientIPOffset) Expect(err).NotTo(HaveOccurred()) @@ -339,27 +339,46 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { serverPod := kickstartPod(cs, serverPodConfig) By("instantiating the client pod") - kickstartPod(cs, clientPodConfig) + clientPod := kickstartPod(cs, clientPodConfig) + + serverInterface, err := getNetworkInterfaceName(serverPod, serverPodConfig, netConfig.name) + Expect(err).NotTo(HaveOccurred(), "failed to extract server pod interface name") + + clientInterface, err := getNetworkInterfaceName(clientPod, clientPodConfig, netConfig.name) + Expect(err).NotTo(HaveOccurred(), "failed to extract client pod interface name") + + // Add external container that will act as external router for the localnet + if (clientPodConfig.usesExternalRouter && len(clientPodConfig.attachments) > 0) || + (serverPodConfig.usesExternalRouter && len(serverPodConfig.attachments) > 0) { + By("instantiating the external container") + externalRouterName, err := createExternalRouter(providerCtx, cs, f, netConfig.vlanID, externalRouterIPOffset) + Expect(err).NotTo(HaveOccurred()) + + By("injecting routes via the external container") + err = injectStaticRoutesViaExternalContainer(f, cs, clientPodConfig, serverPodConfig, + clientInterface, serverInterface, externalRouterName, netConfig.vlanID) + Expect(err).NotTo(HaveOccurred()) + } // Check that the client pod can reach the server pod on the server localnet interface var serverIPs []string - if serverPodConfig.hostNetwork { - serverIPs, err = podIPsFromStatus(cs, serverPodConfig.namespace, serverPodConfig.name) - } else { + if len(serverPodConfig.attachments) > 0 { serverIPs, err = podIPsForAttachment(cs, serverPod.Namespace, serverPod.Name, netConfig.name) - + } else { + serverIPs, err = podIPsFromStatus(cs, serverPodConfig.namespace, serverPodConfig.name) } Expect(err).NotTo(HaveOccurred()) for _, serverIP := range serverIPs { - By(fmt.Sprintf("asserting the *client* can contact the server pod exposed endpoint: %q on port %q", serverIP, port)) curlArgs := []string{} pingArgs := []string{} - if clientPodConfig.attachments != nil { + if len(clientPodConfig.attachments) > 0 { // When the client is attached to a localnet, send probes from the localnet interface - curlArgs = []string{"--interface", "net1"} - pingArgs = []string{"-I", "net1"} + curlArgs = []string{"--interface", clientInterface} + pingArgs = []string{"-I", clientInterface} } + + By(fmt.Sprintf("asserting the *client* can contact the server pod exposed endpoint: %q on port %d", serverIP, port)) Eventually(func() error { return reachServerPodFromClient(cs, serverPodConfig, clientPodConfig, serverIP, port, curlArgs...) }, 2*time.Minute, 6*time.Second).Should(Succeed()) @@ -370,8 +389,34 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { }, 2*time.Minute, 6*time.Second).Should(Succeed()) } }, - ginkgo.Entry( - "can be reached by a client pod in the default network on a different node", + + // The first setup we test is that of a localnet that uses IPs in the host subnet. + // Pod A is a pod in the default network, podL is a pod in a localnet. + // + // +-----------------------+ + // | Kubernetes Node | + // | ovn-worker2 | + // | | + // podA (10.244.1.10/24)---+-------[ br-int ]------+--- podL (172.18.0.4/16, net1) + // (default network) | | | (localnet) + // | [ br-ex ] | + // | 172.18.0.2 | + // | | | + // +-----------|-----------+ + // | + // host network + // 172.18.0.0/16 + // | + // +------------------------------------------+ + // | other hosts / routers / services | + // | (directly reachable in 172.18.0.0/16) | + // +------------------------------------------+ + // + // We test podA when it sits on top of the overlay network, as depicted above, and + // when it is host-networked. + Entry( + // default network -> localnet, different nodes + "can be reached by a client pod in the default network on a different node, when the localnet uses an IP in the host subnet", networkAttachmentConfigParams{ name: secondaryNetworkName, topology: "localnet", @@ -384,36 +429,36 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { attachments: []nadapi.NetworkSelectionElement{{ Name: secondaryNetworkName, }}, - name: podName, - containerCmd: httpServerContainerCmd(port), - needsIPRequestFromHostSubnet: true, // will override attachments above with an IPRequest + name: podName, + containerCmd: httpServerContainerCmd(port), + ipRequestFromSubnet: fromHostSubnet, // override attachments with an IPRequest from host subnet }, false, // scheduled on distinct Nodes - Label("BUG", "OCPBUGS-43004"), ), - ginkgo.Entry( - "can be reached by a client pod in the default network on the same node", + Entry( + // default network -> localnet, same node + "can be reached by a client pod in the default network on the same node, when the localnet uses an IP in the host subnet", networkAttachmentConfigParams{ name: secondaryNetworkName, topology: "localnet", }, podConfiguration{ // client on default network - name: clientPodName + "-same-node", + name: clientPodName, isPrivileged: true, }, podConfiguration{ // server attached to localnet secondary network attachments: []nadapi.NetworkSelectionElement{{ Name: secondaryNetworkName, }}, - name: podName, - containerCmd: httpServerContainerCmd(port), - needsIPRequestFromHostSubnet: true, + name: podName, + containerCmd: httpServerContainerCmd(port), + ipRequestFromSubnet: fromHostSubnet, }, true, // collocated on same Node - Label("BUG", "OCPBUGS-43004"), ), - ginkgo.Entry( - "can reach a host-networked pod on a different node", + Entry( + // localnet -> host network, different nodes + "can reach a host-networked pod on a different node, when the localnet uses an IP in the host subnet", networkAttachmentConfigParams{ name: secondaryNetworkName, topology: "localnet", @@ -422,20 +467,20 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { attachments: []nadapi.NetworkSelectionElement{{ Name: secondaryNetworkName, }}, - name: clientPodName, - isPrivileged: true, - needsIPRequestFromHostSubnet: true, + name: clientPodName, + isPrivileged: true, + ipRequestFromSubnet: fromHostSubnet, }, podConfiguration{ // server on default network, pod is host-networked name: podName, containerCmd: httpServerContainerCmd(port), hostNetwork: true, }, - false, // not collocated on same node - Label("STORY", "SDN-5345"), + false, // not collocated on the same node ), - ginkgo.Entry( - "can reach a host-networked pod on the same node", + Entry( + // localnet -> host network, same node + "can reach a host-networked pod on the same node, when the localnet uses an IP in the host subnet", networkAttachmentConfigParams{ name: secondaryNetworkName, topology: "localnet", @@ -444,17 +489,247 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { attachments: []nadapi.NetworkSelectionElement{{ Name: secondaryNetworkName, }}, - name: clientPodName, - isPrivileged: true, - needsIPRequestFromHostSubnet: true, + name: clientPodName, + isPrivileged: true, + ipRequestFromSubnet: fromHostSubnet, }, podConfiguration{ // server on default network, pod is host-networked name: podName, containerCmd: httpServerContainerCmd(port), hostNetwork: true, }, - true, // collocated on same node - Label("STORY", "SDN-5345"), + true, // collocated on the same node + ), + Entry( + // host network -> localnet, different nodes + "can be reached by a host-networked pod on a different node, when the localnet uses an IP in the host subnet", + networkAttachmentConfigParams{ + name: secondaryNetworkName, + topology: "localnet", + }, + podConfiguration{ // client is host-networked + name: clientPodName, + hostNetwork: true, + isPrivileged: true, + }, + podConfiguration{ // server on localnet + attachments: []nadapi.NetworkSelectionElement{{ + Name: secondaryNetworkName, + }}, + containerCmd: httpServerContainerCmd(port), + name: podName, + ipRequestFromSubnet: fromHostSubnet, + }, + false, // collocated on different nodes + ), + Entry( + // host network -> localnet, same node + "can be reached by a host-networked pod on the same node, when the localnet uses an IP in the host subnet", + networkAttachmentConfigParams{ + name: secondaryNetworkName, + topology: "localnet", + }, + podConfiguration{ // client is host-networked + name: clientPodName, + hostNetwork: true, + isPrivileged: true, + }, + podConfiguration{ // server on localnet + attachments: []nadapi.NetworkSelectionElement{{ + Name: secondaryNetworkName, + }}, + containerCmd: httpServerContainerCmd(port), + name: podName, + ipRequestFromSubnet: fromHostSubnet, + }, + true, // collocated on the same node + ), + // The second setup we test configures: a localnet that uses a VLAN, an external router + // that acts as gateway for the localnet pod for traffic destined to the host network. + // We implement the external router as an external container, where we create a VLAN interface + // on top of eth0 and assign to it an IP in the subnet in use by the localnet. + // Pod A is a pod in the default network, podL is a pod in a localnet. + // + // +-----------------------+ + // | Kubernetes Node | + // | ovn-worker2 | + // | | + // podA (10.244.1.10/24)---+-------[ br-int ]------+--- podL (172.20.0.4/16, net1) + // (default net) | | | (localnet, VLAN 10) + // | [ br-ex ] | + // | 172.18.0.2 | + // +-----------|-----------+ + // | + // host network + // 172.18.0.0/16 + // | + // +------------------------+ + // | external router | + // | | + // | eth0: 172.18.x.x | + // | eth0.10: 172.20.0.55 | + // +------------------------+ + // + // Packet path (ping podA → podL): + // podA (10.244.1.10) + // → br-int + // → br-ex (172.18.0.2, SNAT to node IP) + // → eth0 (external router, 172.18.x.x) + // → eth0.10 (external router, 172.20.0.55) + // → eth0 (external router) + // → br-ex (172.18.0.2) + // → br-int + // → podL (172.20.0.4) + // + // Reply traffic follows the reverse path. + + Entry( + // default network -> localnet, different nodes + "can be reached by a client pod in the default network on a different node, when the localnet uses a VLAN and an external router", + networkAttachmentConfigParams{ + name: secondaryNetworkName, + topology: "localnet", + vlanID: localnetVLANID, + }, + podConfiguration{ // client on default network + name: clientPodName, + isPrivileged: true, + }, + podConfiguration{ // server attached to localnet secondary network + attachments: []nadapi.NetworkSelectionElement{{ + Name: secondaryNetworkName, + }}, + name: podName, + containerCmd: httpServerContainerCmd(port), + ipRequestFromSubnet: fromExternalNetwork, + isPrivileged: true, + usesExternalRouter: true, + }, + false, // scheduled on distinct Nodes + ), + Entry( + // default network -> localnet, same node + "can be reached by a client pod in the default network on the same node, when the localnet uses a VLAN and an external router", + networkAttachmentConfigParams{ + name: secondaryNetworkName, + topology: "localnet", + vlanID: localnetVLANID, + }, + podConfiguration{ // client on default network + name: clientPodName, + isPrivileged: true, + }, + podConfiguration{ // server attached to localnet secondary network + attachments: []nadapi.NetworkSelectionElement{{ + Name: secondaryNetworkName, + }}, + name: podName, + containerCmd: httpServerContainerCmd(port), + ipRequestFromSubnet: fromExternalNetwork, + isPrivileged: true, + usesExternalRouter: true, + }, + true, // scheduled on the same node + ), + Entry( + // host network -> localnet, different nodes + "can be reached by a host-networked pod on a different node, when the localnet uses a VLAN and an external router", + networkAttachmentConfigParams{ + name: secondaryNetworkName, + topology: "localnet", + vlanID: localnetVLANID, + }, + podConfiguration{ // client on host network + name: clientPodName, + hostNetwork: true, + isPrivileged: true, + }, + podConfiguration{ // server attached to localnet secondary network + attachments: []nadapi.NetworkSelectionElement{{ + Name: secondaryNetworkName, + }}, + name: podName, + containerCmd: httpServerContainerCmd(port), + ipRequestFromSubnet: fromExternalNetwork, + isPrivileged: true, + usesExternalRouter: true, + }, + false, // scheduled on distinct Nodes + ), + Entry( + // host network -> localnet, same node + "can be reached by a host-networked pod on the same node, when the localnet uses a VLAN and an external router", + networkAttachmentConfigParams{ + name: secondaryNetworkName, + topology: "localnet", + vlanID: localnetVLANID, + }, + podConfiguration{ // client on host network + name: clientPodName, + hostNetwork: true, + isPrivileged: true, + }, + podConfiguration{ // server attached to localnet secondary network + attachments: []nadapi.NetworkSelectionElement{{ + Name: secondaryNetworkName, + }}, + name: podName, + containerCmd: httpServerContainerCmd(port), + ipRequestFromSubnet: fromExternalNetwork, + isPrivileged: true, + usesExternalRouter: true, + }, + true, // scheduled on the same node + ), + Entry( + // localnet -> host network, different nodes + "can reach a host-network pod on a different node, when the localnet uses a VLAN and an external router", + networkAttachmentConfigParams{ + name: secondaryNetworkName, + topology: "localnet", + vlanID: localnetVLANID, + }, + podConfiguration{ // client attached to localnet secondary network + attachments: []nadapi.NetworkSelectionElement{{ + Name: secondaryNetworkName, + }}, + name: clientPodName, + ipRequestFromSubnet: fromExternalNetwork, + isPrivileged: true, + usesExternalRouter: true, + }, + podConfiguration{ // server on host network + name: podName, + containerCmd: httpServerContainerCmd(port), + hostNetwork: true, + isPrivileged: true, + }, + false, // scheduled on distinct Nodes + ), + Entry( + // localnet -> host network, same node + "can reach a host-network pod on the same node, when the localnet uses a VLAN and an external router", + networkAttachmentConfigParams{ + name: secondaryNetworkName, + topology: "localnet", + vlanID: localnetVLANID, + }, + podConfiguration{ // client attached to localnet secondary network + attachments: []nadapi.NetworkSelectionElement{{ + Name: secondaryNetworkName, + }}, + name: clientPodName, + ipRequestFromSubnet: fromExternalNetwork, + isPrivileged: true, + usesExternalRouter: true, + }, + podConfiguration{ // server on host network + name: podName, + containerCmd: httpServerContainerCmd(port), + hostNetwork: true, + isPrivileged: true, + }, + true, // scheduled on the same node ), ) }) @@ -468,7 +743,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { staticServerIP = "192.168.200.20/24" ) - ginkgo.It("eventually configures pods that were added to an already existing network before the nad", func() { + It("eventually configures pods that were added to an already existing network before the nad", func() { netConfig := newNetworkAttachmentConfig(networkAttachmentConfigParams{ name: secondaryNetworkName, namespace: f.Namespace.Name, @@ -537,7 +812,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { }, 2*time.Minute, 6*time.Second).Should(Equal(v1.PodRunning)) }) - ginkgo.DescribeTable( + DescribeTable( "can communicate over the secondary network", func(netConfigParams networkAttachmentConfigParams, clientPodConfig podConfiguration, serverPodConfig podConfiguration) { netConfig := newNetworkAttachmentConfig(netConfigParams) @@ -561,10 +836,10 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { ) Expect(err).NotTo(HaveOccurred()) - By("Get two scheduable nodes and schedule client and server to be on distinct Nodes") + By("Get two schedulable nodes and schedule client and server to be on distinct Nodes") nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.Background(), f.ClientSet, 2) - framework.ExpectNoError(err, "2 scheduable nodes are required") - Expect(len(nodes.Items)).To(BeNumerically(">=", 1), "cluster should have at least 2 nodes") + framework.ExpectNoError(err, "2 schedulable nodes are required") + Expect(len(nodes.Items)).To(BeNumerically(">", 1), "cluster should have at least 2 nodes") clientPodConfig.nodeSelector = map[string]string{nodeHostnameKey: nodes.Items[0].GetName()} serverPodConfig.nodeSelector = map[string]string{nodeHostnameKey: nodes.Items[1].GetName()} @@ -636,7 +911,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { }, 2*time.Minute, 6*time.Second).Should(Succeed()) } }, - ginkgo.Entry( + Entry( "can communicate over an L2 secondary network when the pods are scheduled in different nodes", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -653,7 +928,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over an L2 - switched - secondary network with `excludeCIDR`s", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -671,7 +946,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over an L3 - routed - secondary network", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -688,7 +963,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over an L3 - routed - secondary network with IPv6 subnet", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -705,7 +980,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over an L3 - routed - secondary network with a dual stack configuration", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -722,7 +997,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over an L2 - switched - secondary network without IPAM", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -740,7 +1015,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { isPrivileged: true, }, ), - ginkgo.Entry( + Entry( "can communicate over an L2 secondary network without IPAM, with static IPs configured via network selection elements", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -762,7 +1037,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over an L2 secondary network with an IPv6 subnet when pods are scheduled in different nodes", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -779,7 +1054,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over an L2 secondary network with a dual stack configuration", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -796,7 +1071,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over a localnet secondary network when the pods are scheduled on different nodes", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -814,7 +1089,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over a localnet secondary network without IPAM when the pods are scheduled on different nodes", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -833,7 +1108,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { isPrivileged: true, }, ), - ginkgo.Entry( + Entry( "can communicate over a localnet secondary network without IPAM when the pods are scheduled on different nodes, with static IPs configured via network selection elements", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -856,7 +1131,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over a localnet secondary network with an IPv6 subnet when pods are scheduled on different nodes", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -874,7 +1149,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { containerCmd: httpServerContainerCmd(port), }, ), - ginkgo.Entry( + Entry( "can communicate over a localnet secondary network with a dual stack configuration when pods are scheduled on different nodes", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1227,7 +1502,6 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { nil, nil, ), - Label("BUG", "OCPBUGS-25928"), ), Entry( "ingress denyall, egress allow all, ingress policy should have no impact on egress", @@ -1393,7 +1667,7 @@ ip a add %[4]s/24 dev %[2]s }, 2*time.Minute, 5*time.Second).Should(BeTrue()) }) - ginkgo.DescribeTable( + DescribeTable( "configure traffic allow lists", func(netConfigParams networkAttachmentConfigParams, allowedClientPodConfig podConfiguration, blockedClientPodConfig podConfiguration, serverPodConfig podConfiguration, policy *mnpapi.MultiNetworkPolicy) { netConfig := newNetworkAttachmentConfig(netConfigParams) @@ -1434,7 +1708,7 @@ ip a add %[4]s/24 dev %[2]s By("asserting the *blocked-client* pod **cannot** contact the server pod exposed endpoint") Expect(connectToServer(blockedClientPodConfig, serverIP, port)).To(MatchError(ContainSubstring("exit code 28"))) }, - ginkgo.Entry( + Entry( "using pod selectors for a pure L2 overlay", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1471,7 +1745,7 @@ ip a add %[4]s/24 dev %[2]s multiNetPolicyPort(port), ), ), - ginkgo.Entry( + Entry( "using pod selectors and port range for a pure L2 overlay", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1509,7 +1783,7 @@ ip a add %[4]s/24 dev %[2]s multiNetPolicyPortRange(port-3, port+5), ), ), - ginkgo.Entry( + Entry( "using pod selectors for a routed topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1546,7 +1820,7 @@ ip a add %[4]s/24 dev %[2]s multiNetPolicyPort(port), ), ), - ginkgo.Entry( + Entry( "using pod selectors for a localnet topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1583,7 +1857,7 @@ ip a add %[4]s/24 dev %[2]s multiNetPolicyPort(port), ), ), - ginkgo.Entry( + Entry( "using IPBlock for a pure L2 overlay", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1615,7 +1889,7 @@ ip a add %[4]s/24 dev %[2]s port, ), ), - ginkgo.Entry( + Entry( "using IPBlock for a routed topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1647,7 +1921,7 @@ ip a add %[4]s/24 dev %[2]s port, ), ), - ginkgo.Entry( + Entry( "using IPBlock for a localnet topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1679,7 +1953,7 @@ ip a add %[4]s/24 dev %[2]s port, ), ), - ginkgo.Entry( + Entry( "using namespace selectors for a pure L2 overlay", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1713,7 +1987,7 @@ ip a add %[4]s/24 dev %[2]s port, ), ), - ginkgo.Entry( + Entry( "using namespace selectors for a routed topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1747,7 +2021,7 @@ ip a add %[4]s/24 dev %[2]s port, ), ), - ginkgo.Entry( + Entry( "using namespace selectors for a localnet topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1782,7 +2056,7 @@ ip a add %[4]s/24 dev %[2]s ), ), - ginkgo.Entry( + Entry( "using IPBlock for an IPAMless pure L2 overlay", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1816,7 +2090,7 @@ ip a add %[4]s/24 dev %[2]s ), ) - ginkgo.DescribeTable( + DescribeTable( "allow all ingress", func(netConfigParams networkAttachmentConfigParams, clientPodConfig podConfiguration, serverPodConfig podConfiguration, policy *mnpapi.MultiNetworkPolicy) { netConfig := newNetworkAttachmentConfig(netConfigParams) @@ -1843,7 +2117,7 @@ ip a add %[4]s/24 dev %[2]s return reachServerPodFromClient(cs, serverPodConfig, clientPodConfig, serverIP, port) }, 2*time.Minute, 6*time.Second).Should(Succeed()) }, - ginkgo.Entry( + Entry( "using ingress allow-all for a localnet topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1875,7 +2149,7 @@ ip a add %[4]s/24 dev %[2]s nil, ), ), - ginkgo.XEntry( + XEntry( "using egress deny-all for a localnet topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1906,9 +2180,8 @@ ip a add %[4]s/24 dev %[2]s nil, nil, ), - Label("BUG", "OCPBUGS-25928"), ), - ginkgo.Entry( + Entry( "using egress deny-all, ingress allow-all for a localnet topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -1944,7 +2217,7 @@ ip a add %[4]s/24 dev %[2]s ), ) - ginkgo.DescribeTable( + DescribeTable( "deny traffic", func(netConfigParams networkAttachmentConfigParams, clientPodConfig podConfiguration, serverPodConfig podConfiguration, policy *mnpapi.MultiNetworkPolicy) { netConfig := newNetworkAttachmentConfig(netConfigParams) @@ -1971,7 +2244,7 @@ ip a add %[4]s/24 dev %[2]s return reachServerPodFromClient(cs, serverPodConfig, clientPodConfig, serverIP, port) }, 2*time.Minute, 6*time.Second).Should(Not(Succeed())) }, - ginkgo.Entry( + Entry( "using ingress deny-all for a localnet topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -2003,7 +2276,7 @@ ip a add %[4]s/24 dev %[2]s nil, ), ), - ginkgo.Entry( + Entry( "using pod selectors and wrong port range for a localnet topology", networkAttachmentConfigParams{ name: secondaryNetworkName, @@ -2113,8 +2386,12 @@ ip a add %[4]s/24 dev %[2]s func kickstartPod(cs clientset.Interface, configuration podConfiguration) *v1.Pod { podNamespacedName := fmt.Sprintf("%s/%s", configuration.namespace, configuration.name) + var ( + pod *v1.Pod + err error + ) By(fmt.Sprintf("instantiating pod %q", podNamespacedName)) - createdPod, err := cs.CoreV1().Pods(configuration.namespace).Create( + _, err = cs.CoreV1().Pods(configuration.namespace).Create( context.Background(), generatePodSpec(configuration), metav1.CreateOptions{}, @@ -2123,13 +2400,15 @@ func kickstartPod(cs clientset.Interface, configuration podConfiguration) *v1.Po By(fmt.Sprintf("asserting that pod %q reaches the `Ready` state", podNamespacedName)) EventuallyWithOffset(1, func() v1.PodPhase { - updatedPod, err := cs.CoreV1().Pods(configuration.namespace).Get(context.Background(), configuration.name, metav1.GetOptions{}) + p, err := cs.CoreV1().Pods(configuration.namespace).Get(context.Background(), configuration.name, metav1.GetOptions{}) if err != nil { return v1.PodFailed } - return updatedPod.Status.Phase + pod = p + return p.Status.Phase + }, 2*time.Minute, 6*time.Second).Should(Equal(v1.PodRunning)) - return createdPod + return pod // return the updated pod } func createNads(f *framework.Framework, nadClient nadclient.K8sCniCncfIoV1Interface, extraNamespace *v1.Namespace, netConfig networkAttachmentConfig) error { @@ -2180,71 +2459,46 @@ func createMultiNetworkPolicy(mnpClient mnpclient.K8sCniCncfIoV1beta1Interface, return err } -func computeIPWithOffset(baseAddr string, increment int) (string, error) { - addr, err := netip.ParsePrefix(baseAddr) +// generateIPsFromNodePrimaryNetworkAddresses returns IPv4 and IPv6 addresses at the provided offset from the primary interface network addresses found on the node +func generateIPsFromNodePrimaryNetworkAddresses(cs clientset.Interface, nodeName string, offset int) ([]string, error) { + hostSubnets, err := getHostSubnetsForNode(cs, nodeName) if err != nil { - return "", fmt.Errorf("Failed to parse CIDR %v", err) + return nil, fmt.Errorf("failed to get host subnets for node %q: %w", nodeName, err) } + return generateIPsFromSubnets(hostSubnets, offset) +} - ip := addr.Addr() - - for i := 0; i < increment; i++ { - ip = ip.Next() - if !ip.IsValid() { - return "", fmt.Errorf("overflow: IP address exceeds bounds") - } +func addIPRequestToPodConfig(cs clientset.Interface, podConfig *podConfiguration, offset int) error { + nodeName, ok := podConfig.nodeSelector[nodeHostnameKey] + if !ok { + return fmt.Errorf("missing node selector %q in podConfig for pod %s/%s", nodeHostnameKey, podConfig.namespace, podConfig.name) } - return netip.PrefixFrom(ip, addr.Bits()).String(), nil -} - -// Given a node name and an offset, generateIPsFromNodePrimaryIfAddr returns an IPv4 and an IPv6 address -// at the provided offset from the primary interface addresses found on the node. -func generateIPsFromNodePrimaryIfAddr(cs clientset.Interface, nodeName string, offset int) ([]string, error) { - var newAddresses []string + var ( + ipsToRequest []string + err error + ) - node, err := cs.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("Failed to get node %s: %v", nodeName, err) - } + switch podConfig.ipRequestFromSubnet { + case fromHostSubnet: + ipsToRequest, err = generateIPsFromNodePrimaryNetworkAddresses(cs, nodeName, offset) - nodeIfAddr, err := util.GetNodeIfAddrAnnotation(node) - if err != nil { - return nil, err - } - nodeAddresses := []string{} - if nodeIfAddr.IPv4 != "" { - nodeAddresses = append(nodeAddresses, nodeIfAddr.IPv4) - } - if nodeIfAddr.IPv6 != "" { - nodeAddresses = append(nodeAddresses, nodeIfAddr.IPv6) - } - for _, nodeAddress := range nodeAddresses { - ipGen, err := ipgenerator.NewIPGenerator(nodeAddress) - if err != nil { - return nil, err - } - newIP, err := ipGen.GenerateIP(offset) - if err != nil { - return nil, err + case fromExternalNetwork: + subnets := filterCIDRs(cs, externalNetworkSubnetV4, externalNetworkSubnetV6) + if len(subnets) == 0 { + return fmt.Errorf("no external network subnets available for IP family support") } - newAddresses = append(newAddresses, newIP.String()) - } - return newAddresses, nil -} + ipsToRequest, err = generateIPsFromSubnets(subnets, offset) -func addIPRequestToPodConfig(cs clientset.Interface, podConfig *podConfiguration, offset int) error { - nodeName, ok := podConfig.nodeSelector[nodeHostnameKey] - if !ok { - return fmt.Errorf("No node selector found on podConfig") + default: + return fmt.Errorf("unknown or unimplemented subnet source: %q", podConfig.ipRequestFromSubnet) } - IPsToRequest, err := generateIPsFromNodePrimaryIfAddr(cs, nodeName, offset) if err != nil { return err } for i := range podConfig.attachments { - podConfig.attachments[i].IPRequest = IPsToRequest + podConfig.attachments[i].IPRequest = ipsToRequest } return nil } diff --git a/test/e2e/multihoming_external_router_utils.go b/test/e2e/multihoming_external_router_utils.go new file mode 100644 index 0000000000..552fadad99 --- /dev/null +++ b/test/e2e/multihoming_external_router_utils.go @@ -0,0 +1,428 @@ +package e2e + +import ( + "context" + "fmt" + "net/netip" + "strings" + + . "github.com/onsi/ginkgo/v2" + + "github.com/ovn-org/ovn-kubernetes/test/e2e/deploymentconfig" + "github.com/ovn-org/ovn-kubernetes/test/e2e/images" + "github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider" + infraapi "github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider/api" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + utilnet "k8s.io/utils/net" +) + +// buildRouteToHostSubnetViaExternalContainer returns ip route add commands to reach the host subnets via the provided gateway IPs +func buildRouteToHostSubnetViaExternalContainer(cs clientset.Interface, nodeName string, gwV4, gwV6, interfaceName string) ([]string, error) { + cmdTemplateV4 := "ip -4 route replace %s via " + gwV4 + " dev " + interfaceName + cmdTemplateV6 := "ip -6 route replace %s via " + gwV6 + " dev " + interfaceName + cmds := []string{} + hostSubnets, err := getHostSubnetsForNode(cs, nodeName) + if err != nil { + return nil, fmt.Errorf("failed to get host subnets for node %s: %w", nodeName, err) + } + for _, hostSubnet := range hostSubnets { + if utilnet.IsIPv4CIDRString(hostSubnet) && gwV4 != "" { + cmds = append(cmds, fmt.Sprintf(cmdTemplateV4, hostSubnet)) + } else if utilnet.IsIPv6CIDRString(hostSubnet) && gwV6 != "" { + cmds = append(cmds, fmt.Sprintf(cmdTemplateV6, hostSubnet)) + } + } + return cmds, nil +} + +// injectRouteViaExternalContainerIntoPod computes and applies host routes inside the given pod to reach +// the host subnets via the external container (VLAN interface), depending on +// the pod's ipRequestFromSubnet field. +func injectRouteViaExternalContainerIntoPod(f *framework.Framework, cs clientset.Interface, podConfig podConfiguration, + podInterfaceName, externalContainerName string, vlanID int) error { + + nodeName, ok := podConfig.nodeSelector[nodeHostnameKey] + if !ok { + return fmt.Errorf("nodeSelector should contain %s key", nodeHostnameKey) + } + + cmds := []string{} + vlanIface := fmt.Sprintf("%s.%d", "eth0", vlanID) + gwV4IPs, gwV6IPs, err := getExternalContainerInterfaceIPs(externalContainerName, vlanIface) + if err != nil { + return fmt.Errorf("failed to get external container interface IPs: %w", err) + } + if len(gwV4IPs) == 0 && len(gwV6IPs) == 0 { + return fmt.Errorf("no IPs found on VLAN interface %s of external container %s", vlanIface, externalContainerName) + } + + // Normalize IP addresses by removing CIDR notation if present + gwV4IPs, err = normalizeIPAddresses(gwV4IPs) + if err != nil { + return fmt.Errorf("failed to normalize IPv4 addresses from external container interface: %w", err) + } + gwV6IPs, err = normalizeIPAddresses(gwV6IPs) + if err != nil { + return fmt.Errorf("failed to normalize IPv6 addresses from external container interface: %w", err) + } + + // Take the first container IP as gateway. + var gwIPV4, gwIPV6 string + if len(gwV4IPs) > 0 { + gwIPV4 = gwV4IPs[0] + } + + for _, ip := range gwV6IPs { + if addr, err := netip.ParseAddr(ip); err == nil && !addr.IsLinkLocalUnicast() { + gwIPV6 = ip + break + } + } + + cmds, err = buildRouteToHostSubnetViaExternalContainer(cs, nodeName, gwIPV4, gwIPV6, podInterfaceName) + if err != nil { + return fmt.Errorf("failed to build route to host subnet via external container: %w", err) + } + + for _, cmd := range cmds { + framework.Logf("Adding to pod %s/%s route to host subnet via external container %s: %s", podConfig.namespace, podConfig.name, externalContainerName, cmd) + _, stderr, err := ExecShellInPodWithFullOutput(f, podConfig.namespace, podConfig.name, cmd) + if err != nil || stderr != "" { + return fmt.Errorf("failed to add route to external container (cmd=%s): stderr=%s, err=%w\n", cmd, stderr, err) + } + } + + return nil +} + +// createExternalRouter creates an external container that acts as a router for localnet testing +func createExternalRouter(providerCtx infraapi.Context, cs clientset.Interface, f *framework.Framework, vlanID, ipOffset int) (string, error) { + // Add external container that will act as external router for the localnet + primaryProviderNetwork, err := infraprovider.Get().PrimaryNetwork() + if err != nil { + return "", fmt.Errorf("failed to get primary provider network: %w", err) + } + externalContainerName := f.Namespace.Name + "-external-router" + + routerSubnets := filterCIDRs(cs, externalNetworkSubnetV4, externalNetworkSubnetV6) + routerIPs, err := generateIPsFromSubnets(routerSubnets, ipOffset) + if err != nil { + return "", fmt.Errorf("failed to generate IP for external router: %w", err) + } + if len(routerIPs) == 0 { + return "", fmt.Errorf("no supported IP families found for the external router") + } + + // - create a VLAN interface on top of eth0. + // - assign the generated IP to the VLAN interface. + // - enable IP forwarding. + // - sleep to keep the container running. + var commandBuilder strings.Builder + commandBuilder.WriteString(fmt.Sprintf("ip link add link eth0 name eth0.%d type vlan id %d; ", vlanID, vlanID)) + for _, ip := range routerIPs { + commandBuilder.WriteString(fmt.Sprintf("ip addr add %s dev eth0.%d; ", ip, vlanID)) + } + commandBuilder.WriteString(fmt.Sprintf("ip link set eth0.%d up; ", vlanID)) + commandBuilder.WriteString("sysctl -w net.ipv4.ip_forward=1; ") + commandBuilder.WriteString("sysctl -w net.ipv6.conf.all.forwarding=1; ") + commandBuilder.WriteString("sleep infinity") + + externalContainerSpec := infraapi.ExternalContainer{ + Name: externalContainerName, + Image: images.AgnHost(), + Network: primaryProviderNetwork, + Entrypoint: "bash", + CmdArgs: []string{"-c", commandBuilder.String()}, + } + + _, err = providerCtx.CreateExternalContainer(externalContainerSpec) + if err != nil { + return "", fmt.Errorf("failed to create external router container: %w", err) + } + + return externalContainerName, nil +} + +// injectStaticRoutesViaExternalContainer configures the localnet pod to reach the host subnet and +// the hosts/OVN to reach the localnet subnet. +// We need to inject static routes in the following places: +// +// 1. on the localnet pod we need a route to reach the host subnet +// via the VLAN interface of the external container; +// +// 2. in NBDB, if the cluster is in shared gateway mode, we need a route that tells +// OVN to route traffic to the localnet subnet via the external container IPs; +// +// 3. in the host routing table, if the cluster is in local gateway mode, we need a route +// that tells the host to route traffic to the localnet subnet via the external container IPs. +// We need this also for host-networked pods to reach the localnet subnet regardless of the gateway mode. +func injectStaticRoutesViaExternalContainer(f *framework.Framework, cs clientset.Interface, + clientPodConfig, serverPodConfig podConfiguration, clientInterface, serverInterface, externalContainerName string, vlanID int) error { + if clientPodConfig.usesExternalRouter && len(clientPodConfig.attachments) > 0 { + if err := injectRouteViaExternalContainerIntoPod(f, cs, clientPodConfig, clientInterface, externalContainerName, vlanID); err != nil { + return fmt.Errorf("failed to add route to client pod %s/%s: %w", clientPodConfig.namespace, clientPodConfig.name, err) + } + } + + if serverPodConfig.usesExternalRouter && len(serverPodConfig.attachments) > 0 { + if err := injectRouteViaExternalContainerIntoPod(f, cs, serverPodConfig, serverInterface, externalContainerName, vlanID); err != nil { + return fmt.Errorf("failed to add route to server pod %s/%s: %w", serverPodConfig.namespace, serverPodConfig.name, err) + } + } + + if err := injectStaticRoutesIntoNodes(f, cs, externalContainerName); err != nil { + return fmt.Errorf("failed to add static routes into nodes: %w", err) + } + return nil +} + +// injectStaticRoutesIntoNodes adds routes for externalNetworkSubnetV4/V6 +// via the external container IPs on the primary provider network. +// The type of routes differs according to the OVNK architecture (interconnect vs centralized) +// and the gateway mode: +// | | Local GW | Shared GW | +// | IC | linux route on all node | linux routes on all nodes; OVN routes on all nodes for the local GW router | +// | non-IC | linux routes on all nodes | linux routes on all nodes; OVN routes on NBDB leader for all GW routers | +func injectStaticRoutesIntoNodes(f *framework.Framework, cs clientset.Interface, externalContainerName string) error { + framework.Logf("Injecting Linux kernel routes for host-networked pods (and for OVN pods when in local gateway mode)") + if err := injectRoutesWithCommandBuilder(f, cs, externalContainerName, hostRoutingTableCommandBuilder{}); err != nil { + return err + } + + if !IsGatewayModeLocal(cs) { + framework.Logf("Shared gateway mode: injecting OVN routes for overlay pods") + if err := injectRoutesWithCommandBuilder(f, cs, externalContainerName, ovnLogicalRouterCommandBuilder{}); err != nil { + return err + } + } + + return nil +} + +// routeCommand represents a route add/delete command for a specific gateway mode +type routeCommand struct { + addCmd []string + deleteCmd []string + logMsg string + target string // what we're adding the route to (logical router name or node name) +} + +// routeCommandBuilder defines the interface for building gateway-mode-specific route commands +type routeCommandBuilder interface { + buildRouteCommand(nodeName, cidr, nextHop string) routeCommand +} + +// ovnLogicalRouterCommandBuilder builds commands for OVN logical routes (shared gateway mode) +type ovnLogicalRouterCommandBuilder struct{} + +func (b ovnLogicalRouterCommandBuilder) buildRouteCommand(nodeName, cidr, nextHop string) routeCommand { + logicalRouterName := "GR_" + nodeName + return routeCommand{ + addCmd: []string{"ovn-nbctl", "--may-exist", "--", "lr-route-add", logicalRouterName, cidr, nextHop}, + deleteCmd: []string{"ovn-nbctl", "--if-exists", "--", "lr-route-del", logicalRouterName, cidr}, + logMsg: fmt.Sprintf("OVN logical router route %s via %s to %s", cidr, nextHop, logicalRouterName), + target: logicalRouterName, + } +} + +// hostRoutingTableCommandBuilder builds commands for routes on the host +type hostRoutingTableCommandBuilder struct{} + +func (b hostRoutingTableCommandBuilder) buildRouteCommand(nodeName, cidr, nextHop string) routeCommand { + return routeCommand{ + addCmd: []string{"ip", "route", "replace", cidr, "via", nextHop}, + deleteCmd: []string{"ip", "route", "del", cidr, "via", nextHop}, + logMsg: fmt.Sprintf("host route %s via %s to node %s", cidr, nextHop, nodeName), + target: nodeName, + } +} + +// getOvnKubePodsForRouteInjection determines which pods to use for route injection based on the command builder type +// and cluster configuration. +func getOvnKubePodsForRouteInjection(f *framework.Framework, cs clientset.Interface, cmdBuilder routeCommandBuilder) (*v1.PodList, error) { + ovnKubernetesNamespace := deploymentconfig.Get().OVNKubernetesNamespace() + + var podList *v1.PodList + var err error + + if _, isHostRoutingCommand := cmdBuilder.(hostRoutingTableCommandBuilder); isHostRoutingCommand { + framework.Logf("Host routing command: selecting all ovnkube-node pods") + podList, err = cs.CoreV1().Pods(ovnKubernetesNamespace).List(context.TODO(), metav1.ListOptions{LabelSelector: "name=ovnkube-node"}) + } else if isInterconnectEnabled() { + framework.Logf("OVN command with interconnect: selecting all OVN DB pods") + podList, err = cs.CoreV1().Pods(ovnKubernetesNamespace).List(context.TODO(), metav1.ListOptions{LabelSelector: "ovn-db-pod=true"}) + } else { + framework.Logf("OVN command without interconnect: selecting DB leader pod") + leaderPod, findErr := findOVNDBLeaderPod(f, cs, ovnKubernetesNamespace) + if findErr != nil { + return nil, fmt.Errorf("failed to find OVN DB leader pod: %w", findErr) + } + podList = &v1.PodList{Items: []v1.Pod{*leaderPod}} + } + + if err != nil { + return nil, err + } + + if len(podList.Items) == 0 { + return nil, fmt.Errorf("no ovnkube pods found to execute route commands") + } + + return podList, nil +} + +// getTargetNodesForRouteInjection determines which nodes should be targeted for route injection +// based on the command builder type and cluster configuration. +func getTargetNodesForRouteInjection(cs clientset.Interface, cmdBuilder routeCommandBuilder, nodeName string) ([]string, error) { + // For host routing commands, always target the current pod's node + if _, isHostRoutingCommand := cmdBuilder.(hostRoutingTableCommandBuilder); isHostRoutingCommand { + return []string{nodeName}, nil + } + + // OVN routes + if isInterconnectEnabled() { + return []string{nodeName}, nil // each pod targets its own gateway router + } + + // non-interconnect mode: DB pod targets all gateway routers + allNodes, err := cs.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list nodes: %w", err) + } + + var targetNodeNames []string + for _, node := range allNodes.Items { + targetNodeNames = append(targetNodeNames, node.Name) + } + return targetNodeNames, nil +} + +// routeInfo represents a route destination and next-hop pair +type routeInfo struct { + destination string + nextHop string +} + +// buildRouteList creates a list of routes based on available IP families and gateways +func buildRouteList(clientSet clientset.Interface, v4gateway, v6gateway string) ([]routeInfo, error) { + var routes []routeInfo + + if isIPv4Supported(clientSet) && v4gateway != "" { + routes = append(routes, routeInfo{ + destination: externalNetworkSubnetV4, + nextHop: v4gateway, + }) + } + + if isIPv6Supported(clientSet) && v6gateway != "" { + routes = append(routes, routeInfo{ + destination: externalNetworkSubnetV6, + nextHop: v6gateway, + }) + } + + if len(routes) == 0 { + return nil, fmt.Errorf("no routes to inject (check IP families and external container addresses)") + } + + return routes, nil +} + +// podExecutionContext holds pod-specific information for route execution +type podExecutionContext struct { + pod v1.Pod + containerName string + targetNodes []string + cmdBuilder routeCommandBuilder +} + +func newPodExecutionContext(cs clientset.Interface, cmdBuilder routeCommandBuilder, pod v1.Pod) (*podExecutionContext, error) { + targetNodes, err := getTargetNodesForRouteInjection(cs, cmdBuilder, pod.Spec.NodeName) + if err != nil { + return nil, fmt.Errorf("failed to get target nodes for pod %s: %w", pod.Name, err) + } + + return &podExecutionContext{ + pod: pod, + containerName: pod.Spec.Containers[0].Name, + targetNodes: targetNodes, + cmdBuilder: cmdBuilder, + }, nil +} + +func (ctx *podExecutionContext) executeAllRoutes(f *framework.Framework, routes []routeInfo) error { + for _, route := range routes { + for _, targetNode := range ctx.targetNodes { + routeCmd := ctx.cmdBuilder.buildRouteCommand(targetNode, route.destination, route.nextHop) + + if err := addRoute(f, ctx.pod.Namespace, ctx.pod.Name, ctx.containerName, routeCmd); err != nil { + return err + } + + scheduleRouteCleanup(f, ctx.pod.Namespace, ctx.pod.Name, ctx.containerName, routeCmd) + } + } + return nil +} + +func addRoute(f *framework.Framework, namespace, podName, containerName string, routeCmd routeCommand) error { + stdout, stderr, err := ExecCommandInContainerWithFullOutput(f, namespace, podName, containerName, routeCmd.addCmd...) + if err != nil || stderr != "" { + return fmt.Errorf("failed to add %s (pod=%s, container=%s): stdout=%q, stderr=%q, cmd=%v: %w", + routeCmd.logMsg, podName, containerName, stdout, stderr, routeCmd.addCmd, err) + } + framework.Logf("Successfully added %s", routeCmd.logMsg) + return nil +} + +func scheduleRouteCleanup(f *framework.Framework, namespace, podName, containerName string, routeCmd routeCommand) { + DeferCleanup(func() { + _, stderr, err := ExecCommandInContainerWithFullOutput(f, namespace, podName, containerName, routeCmd.deleteCmd...) + if err != nil { + framework.Logf("Warning: Failed to delete route from %s (cmd=%s): %v, stderr: %s", + routeCmd.target, routeCmd.deleteCmd, err, stderr) + } + }) +} + +func injectRoutesWithCommandBuilder(f *framework.Framework, cs clientset.Interface, externalContainerName string, cmdBuilder routeCommandBuilder) error { + primaryProviderNetwork, err := infraprovider.Get().PrimaryNetwork() + if err != nil { + return fmt.Errorf("failed to get primary network: %w", err) + } + + v4gateway, v6gateway, err := getExternalContainerInterfaceIPsOnNetwork(externalContainerName, primaryProviderNetwork.Name()) + if err != nil { + return fmt.Errorf("failed to get external container interface IPs on provider network: %w", err) + } + + // Build list of routes to inject + routes, err := buildRouteList(f.ClientSet, v4gateway, v6gateway) + if err != nil { + return err + } + + // Get target pods for route injection + ovnkubePods, err := getOvnKubePodsForRouteInjection(f, cs, cmdBuilder) + if err != nil { + return fmt.Errorf("failed to select target pods: %w", err) + } + + // Execute route commands for each pod + for _, pod := range ovnkubePods.Items { + podCtx, err := newPodExecutionContext(cs, cmdBuilder, pod) + if err != nil { + return err + } + + if err := podCtx.executeAllRoutes(f, routes); err != nil { + return err + } + } + + return nil +} diff --git a/test/e2e/multihoming_utils.go b/test/e2e/multihoming_utils.go index 816f7cccd1..bf411a3d38 100644 --- a/test/e2e/multihoming_utils.go +++ b/test/e2e/multihoming_utils.go @@ -24,6 +24,8 @@ import ( mnpapi "github.com/k8snetworkplumbingwg/multi-networkpolicy/pkg/apis/k8s.cni.cncf.io/v1beta1" nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" utilnet "k8s.io/utils/net" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/generator/ip" ) func netCIDR(netCIDR string, netPrefixLengthPerNode int) string { @@ -72,7 +74,7 @@ func filterIPs(cs clientset.Interface, ips ...string) []string { } func filterIPsAndJoin(cs clientset.Interface, ips string) string { - return joinStrings(filterIPs(cs, strings.Split(ips, ",")...)...) + return joinStrings(filterIPs(cs, strings.Split(ips, ",")...)...) } func getNetCIDRSubnet(netCIDR string) (string, error) { @@ -190,16 +192,17 @@ func patchNADSpec(nadClient nadclient.K8sCniCncfIoV1Interface, name, namespace s } type podConfiguration struct { - attachments []nadapi.NetworkSelectionElement - containerCmd []string - name string - namespace string - nodeSelector map[string]string - isPrivileged bool - labels map[string]string - requiresExtraNamespace bool - hostNetwork bool - needsIPRequestFromHostSubnet bool + attachments []nadapi.NetworkSelectionElement + containerCmd []string + name string + namespace string + nodeSelector map[string]string + isPrivileged bool + labels map[string]string + requiresExtraNamespace bool + hostNetwork bool + ipRequestFromSubnet string + usesExternalRouter bool } func generatePodSpec(config podConfiguration) *v1.Pod { @@ -355,7 +358,7 @@ func pingServer(clientPodConfig podConfiguration, serverIP string, args ...strin clientPodConfig.name, "--", "ping", - "-c", "1", // send one ICMP echo request + "-c", "3", // send three ICMP echo requests "-W", "2", // timeout after 2 seconds if no response } baseArgs = append(baseArgs, args...) @@ -405,7 +408,11 @@ func podIPsForAttachment(k8sClient clientset.Interface, podNamespace string, pod if err != nil { return nil, err } - if len(netStatus) != 1 { + + if len(netStatus) == 0 { + return nil, fmt.Errorf("no status entry for attachment %s on pod %s", attachmentName, namespacedName(podNamespace, podName)) + } + if len(netStatus) > 1 { return nil, fmt.Errorf("more than one status entry for attachment %s on pod %s", attachmentName, namespacedName(podNamespace, podName)) } if len(netStatus[0].IPs) == 0 { @@ -768,3 +775,27 @@ func getPodAnnotationIPsForAttachmentByIndex(k8sClient clientset.Interface, podN } return ipnets[index].IP.String(), nil } + +// generateIPsFromSubnets generates IP addresses from the given subnets with the specified offset +func generateIPsFromSubnets(subnets []string, offset int) ([]string, error) { + var addrs []string + for _, s := range subnets { + s = strings.TrimSpace(s) + if s == "" { + continue + } + ipGen, err := ip.NewIPGenerator(s) + if err != nil { + return nil, err + } + ip, err := ipGen.GenerateIP(offset) + if err != nil { + return nil, err + } + addrs = append(addrs, ip.String()) + } + if len(addrs) == 0 { + return nil, fmt.Errorf("no valid subnets provided") + } + return addrs, nil +} diff --git a/test/e2e/util.go b/test/e2e/util.go index f9672f71e9..569674597a 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "net" + "net/netip" "os" "path/filepath" "regexp" @@ -14,6 +15,7 @@ import ( "text/template" "time" + nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -823,6 +825,71 @@ func assertACLLogs(targetNodeName string, policyNameRegex string, expectedACLVer return false, nil } +// getExternalContainerInterfaceIPsOnNetwork returns the IPv4 and IPv6 addresses (if any) +// of the given external container on the specified provider network. +func getExternalContainerInterfaceIPsOnNetwork(containerName, networkName string) (string, string, error) { + netw, err := infraprovider.Get().GetNetwork(networkName) + if err != nil { + return "", "", fmt.Errorf("failed to get provider network %q: %w", networkName, err) + } + ni, err := infraprovider.Get().GetExternalContainerNetworkInterface( + infraapi.ExternalContainer{Name: containerName}, + netw, + ) + if err != nil { + return "", "", fmt.Errorf("failed to get network interface for container %q on network %q: %w", containerName, netw.Name(), err) + } + return ni.IPv4, ni.IPv6, nil +} + +// getExternalContainerInterfaceIPs returns IPv4 and IPv6 addresses configured +// on the given interface inside the given external container. This is useful +// for manually-configured interfaces like VLAN interfaces. +func getExternalContainerInterfaceIPs(containerName, ifaceName string) ([]string, []string, error) { + container := infraapi.ExternalContainer{Name: containerName} + + // Replicates the relevant fields from the json output by "ip -j addr show" + type addrInfo struct { + Family string `json:"family"` + Local string `json:"local"` + Scope string `json:"scope"` + } + type ipAddrJSON struct { + AddrInfo []addrInfo `json:"addr_info"` + } + + out, err := infraprovider.Get().ExecExternalContainerCommand( + container, []string{"ip", "-j", "addr", "show", "dev", ifaceName}) + if err != nil { + return nil, nil, fmt.Errorf("failed to exec on container %q: %w", containerName, err) + } + var parsed []ipAddrJSON + if err := json.Unmarshal([]byte(out), &parsed); err != nil { + return nil, nil, fmt.Errorf("failed to parse ip -j output: %w", err) + } + + var v4, v6 []string + for _, entry := range parsed { + for _, ai := range entry.AddrInfo { + if ai.Local == "" { + continue + } + // Skip link-local/host-scoped addresses + if ai.Scope == "link" || ai.Scope == "host" { + continue + } + switch ai.Family { + case "inet": + v4 = append(v4, ai.Local) + case "inet6": + v6 = append(v6, ai.Local) + } + } + } + + return v4, v6, nil +} + // patchServiceStringValue patches service serviceName in namespace serviceNamespace with provided string value. func patchServiceStringValue(c kubernetes.Interface, serviceName, serviceNamespace, jsonPath, value string) error { patch := []struct { @@ -1534,3 +1601,151 @@ func isDNSNameResolverEnabled() bool { val, present := os.LookupEnv("OVN_ENABLE_DNSNAMERESOLVER") return present && val == "true" } + +// Given a node name, returns the host subnets (IPv4/IPv6) of the node primary interface +// as annotated by OVN-Kubernetes. The returned slice may contain zero, one, or two CIDRs. +func getHostSubnetsForNode(cs clientset.Interface, nodeName string) ([]string, error) { + node, err := cs.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get node %s: %v", nodeName, err) + } + nodeIfAddr, err := util.GetNodeIfAddrAnnotation(node) + if err != nil { + return nil, err + } + hostSubnets := []string{} + if nodeIfAddr.IPv4 != "" { + ip, ipNet, err := net.ParseCIDR(nodeIfAddr.IPv4) + if err != nil { + return nil, fmt.Errorf("failed to parse IPv4 address %s: %v", nodeIfAddr.IPv4, err) + } + ipNet.IP = ip.Mask(ipNet.Mask) + hostSubnets = append(hostSubnets, ipNet.String()) + } + if nodeIfAddr.IPv6 != "" { + ip, ipNet, err := net.ParseCIDR(nodeIfAddr.IPv6) + if err != nil { + return nil, fmt.Errorf("failed to parse IPv6 address %s: %v", nodeIfAddr.IPv6, err) + } + ipNet.IP = ip.Mask(ipNet.Mask) + hostSubnets = append(hostSubnets, ipNet.String()) + } + return hostSubnets, nil +} + +// normalizeIP removes CIDR notation from an IP address if present and validates/normalizes the IP format. +// For example, "10.0.0.2/24" becomes "10.0.0.2". +func normalizeIP(s string) (string, error) { + if s == "" { + return s, nil + } + if p, err := netip.ParsePrefix(s); err == nil { + return p.Addr().String(), nil + } + if a, err := netip.ParseAddr(s); err == nil { + return a.String(), nil + } + return "", fmt.Errorf("invalid IP address: %s", s) +} + +func normalizeIPAddresses(ips []string) ([]string, error) { + normalized := make([]string, len(ips)) + for i, ip := range ips { + normalizedIP, err := normalizeIP(ip) + if err != nil { + return nil, fmt.Errorf("failed to normalize IP addresses: %w", err) + } + normalized[i] = normalizedIP + } + return normalized, nil +} + +// getNetworkInterfaceName extracts the interface name from a pod's network-status annotation +// If the pod is host-networked, it returns eth0. +// If the pod has attachments, it finds the interface for the specified network +// If the pod has no attachments, it returns the default network interface +func getNetworkInterfaceName(pod *v1.Pod, podConfig podConfiguration, netConfigName string) (string, error) { + var predicate func(nadapi.NetworkStatus) bool + if podConfig.hostNetwork { + return "eth0", nil + } + if len(podConfig.attachments) > 0 { + // Pod has attachments - find the specific network interface + expectedNetworkName := fmt.Sprintf("%s/%s", podConfig.namespace, netConfigName) + predicate = func(status nadapi.NetworkStatus) bool { + return status.Name == expectedNetworkName + } + } else { + // Pod has no attachments - find the default network interface + predicate = func(status nadapi.NetworkStatus) bool { + return status.Name == "ovn-kubernetes" || status.Default + } + } + networkStatuses, err := podNetworkStatus(pod, predicate) + if err != nil { + return "", fmt.Errorf("failed to get network status from pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + if len(networkStatuses) == 0 { + if len(podConfig.attachments) > 0 { + return "", fmt.Errorf("no network interface found for network %s/%s", podConfig.namespace, netConfigName) + } + return "", fmt.Errorf("no default network interface found") + } + if len(networkStatuses) > 1 { + return "", fmt.Errorf("multiple network interfaces found matching criteria") + } + iface := networkStatuses[0].Interface + // Multus may omit Interface for the default network; default to eth0. + if iface == "" && len(podConfig.attachments) == 0 { + return "eth0", nil + } + return iface, nil +} + +// findOVNDBLeaderPod finds the ovnkube-db pod that is currently the northbound database leader +func findOVNDBLeaderPod(f *framework.Framework, cs clientset.Interface, namespace string) (*v1.Pod, error) { + dbPods, err := cs.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: "ovn-db-pod=true"}) + if err != nil { + return nil, fmt.Errorf("failed to list ovnkube-db pods: %v", err) + } + + if len(dbPods.Items) == 0 { + return nil, fmt.Errorf("no ovnkube-db pods found") + } + + if len(dbPods.Items) == 1 { + return &dbPods.Items[0], nil + } + + for i := range dbPods.Items { + pod := &dbPods.Items[i] + if pod.Status.Phase != v1.PodRunning { + continue + } + + stdout, stderr, err := ExecCommandInContainerWithFullOutput(f, namespace, pod.Name, "nb-ovsdb", + "ovsdb-client", "query", "unix:/var/run/openvswitch/ovnnb_db.sock", + `["_Server", {"op":"select", "table":"Database", "where":[["name", "==", "OVN_Northbound"]], "columns": ["leader"]}]`) + + if err != nil { + framework.Logf("Warning: Failed to query leader status on pod %s: %v, stderr: %s", pod.Name, err, stderr) + continue + } + + // Parse the JSON response to check if this pod is the leader + // Expected: [{"rows":[{"leader":true}]}] + type dbResp struct { + Rows []struct { + Leader bool `json:"leader"` + } `json:"rows"` + } + var resp []dbResp + if err := json.Unmarshal([]byte(stdout), &resp); err == nil && + len(resp) > 0 && len(resp[0].Rows) > 0 && resp[0].Rows[0].Leader { + framework.Logf("Found nbdb leader pod: %s", pod.Name) + return pod, nil + } + } + + return nil, fmt.Errorf("no nbdb leader pod found among %d ovnkube-db pods", len(dbPods.Items)) +}