diff --git a/test/extended/networking/egressip.go b/test/extended/networking/egressip.go index 47a695e181bc..76ae88e83343 100644 --- a/test/extended/networking/egressip.go +++ b/test/extended/networking/egressip.go @@ -742,35 +742,40 @@ func applyEgressIPObject(oc *exutil.CLI, cloudNetworkClientset cloudnetwork.Inte _, err := runOcWithRetry(oc.AsAdmin(), "apply", "-f", egressIPYamlPath) o.Expect(err).NotTo(o.HaveOccurred()) - framework.Logf(fmt.Sprintf("Waiting for CloudPrivateIPConfig creation for a maximum of %d seconds", timeout)) - var exists bool - var isAssigned bool - o.Eventually(func() bool { - for eip := range egressIPSet { - exists, isAssigned, err = cloudPrivateIpConfigExists(oc, cloudNetworkClientset, eip) - o.Expect(err).NotTo(o.HaveOccurred()) - if !exists { - framework.Logf("CloudPrivateIPConfig for %s not found.", eip) - return false - } - if !isAssigned { - framework.Logf("CloudPrivateIPConfig for %s not assigned.", eip) - return false + if cloudNetworkClientset != nil { + framework.Logf(fmt.Sprintf("Waiting for CloudPrivateIPConfig creation for a maximum of %d seconds", timeout)) + var exists bool + var isAssigned bool + o.Eventually(func() bool { + for eip := range egressIPSet { + exists, isAssigned, err = cloudPrivateIpConfigExists(oc, cloudNetworkClientset, eip) + o.Expect(err).NotTo(o.HaveOccurred()) + if !exists { + framework.Logf("CloudPrivateIPConfig for %s not found.", eip) + return false + } + if !isAssigned { + framework.Logf("CloudPrivateIPConfig for %s not assigned.", eip) + return false + } } - } - framework.Logf("CloudPrivateIPConfigs for %v found.", egressIPSet) - return true - }, time.Duration(timeout)*time.Second, 5*time.Second).Should(o.BeTrue()) + framework.Logf("CloudPrivateIPConfigs for %v found.", egressIPSet) + return true + }, time.Duration(timeout)*time.Second, 5*time.Second).Should(o.BeTrue()) + } framework.Logf(fmt.Sprintf("Waiting for EgressIP addresses inside status of EgressIP CR %s for a maximum of %d seconds", egressIPObjectName, timeout)) var hasIP bool + var nodeName string o.Eventually(func() bool { for eip := range egressIPSet { - hasIP, err = egressIPStatusHasIP(oc, egressIPObjectName, eip) + hasIP, nodeName, err = egressIPStatusHasIP(oc, egressIPObjectName, eip) o.Expect(err).NotTo(o.HaveOccurred()) if !hasIP { framework.Logf("EgressIP object %s does not have IP %s in its status field.", egressIPObjectName, eip) return false + } else { + egressIPSet[eip] = nodeName } } framework.Logf("Egress IP object %s does have all IPs for %v.", egressIPObjectName, egressIPSet) diff --git a/test/extended/networking/egressip_helpers.go b/test/extended/networking/egressip_helpers.go index 3e0ccef8939b..3f5c21e9f367 100644 --- a/test/extended/networking/egressip_helpers.go +++ b/test/extended/networking/egressip_helpers.go @@ -404,7 +404,7 @@ func createPacketSnifferDaemonSet(oc *exutil.CLI, namespace string, scheduleOnHo } var ds *appsv1.DaemonSet - retries := 12 + retries := 48 pollInterval := 5 for i := 0; i < retries; i++ { // Get the DS @@ -415,7 +415,9 @@ func createPacketSnifferDaemonSet(oc *exutil.CLI, namespace string, scheduleOnHo // Check if NumberReady == DesiredNumberScheduled. // In that case, simply return as all went well. - if ds.Status.NumberReady == ds.Status.DesiredNumberScheduled { + if ds.Status.NumberReady == ds.Status.DesiredNumberScheduled && + ds.Status.CurrentNumberScheduled == ds.Status.DesiredNumberScheduled && + ds.Status.DesiredNumberScheduled > 0 { return ds, nil } // If no port conflict error was found, simply sleep for pollInterval and then @@ -426,15 +428,14 @@ func createPacketSnifferDaemonSet(oc *exutil.CLI, namespace string, scheduleOnHo // The DaemonSet is not ready, but this is not because of a port conflict. // This shouldn't happen and other parts of the code will likely report this error // as a CI failure. - return ds, fmt.Errorf("Daemonset still not ready after %d tries", retries) + return ds, fmt.Errorf("Daemonset still not ready after %d tries: ready=%d, scheduled=%d, desired=%d", retries, ds.Status.NumberReady, ds.Status.CurrentNumberScheduled, ds.Status.DesiredNumberScheduled) } const ( // The tcpCaptureScript runs tcpdump and extracts all GET request strings from the packets. // The resulting lines will be something like: - // 10.128.2.15.36749 /f8f721fa-53c9-444f-bc96-69c7388fcb5a - tcpCaptureScript = `#!/bin/bash -tcpdump -nn -i %s -l -s 0 -A 'tcp and port %d' | awk '/IP / || /IP6 / {ip=$3} /GET \// {print ip, $2}' + // Parsed 05:38:34.307832 10.128.2.15.36749 /f8f721fa-53c9-444f-bc96-69c7388fcb5a + tcpCaptureScript = `tcpdump -nn -i %s -l -s 0 -A 'tcp and port %d' | awk 'match($0,/IP6?[[:space:]]+([0-9a-fA-F:\.]+[0-9a-fA-F])/,arr) {ts=$1; ip=arr[1]} $0 !~ /HTTP.*GET/ && match($0,/GET[[:space:]]+([^[:space:]]+)/,arr) {print "Parsed", ts, ip, arr[1]} // {print $0}' ` // The udpCaptureScript runs tcpdump with option -xx and then decodes the hexadecimal information. @@ -443,8 +444,8 @@ tcpdump -nn -i %s -l -s 0 -A 'tcp and port %d' | awk '/IP / || /IP6 / {ip=$3} /G // that's captured). // tshark would definitely be the better tool here, but that would introduce another dependency. Hence, // decode the hexadecimal information and look for payload that is marked with 'START(.*)EOF$' and extract - // the '(.*)' part. The resulting lines will be `sourceIP + " " + z.group(1)`, hence something like: - // 10.128.2.15.36749 f8f721fa-53c9-444f-bc96-69c7388fcb5a + // the '(.*)' part. The resulting lines will be `"Parsed " + timestamp + " " + sourceIP + " " + z.group(1) + "_" + z.group(2)`, hence something like: + // Parsed 05:38:34.307832 10.128.2.15.36749 f8f721fa-53c9-444f-bc96-69c7388fcb5a_1 udpCaptureScript = `#!/bin/bash cat <<'EOF' > capture-python.py @@ -466,6 +467,7 @@ udpPayloadOffset = 0 # globals fullHex = [] sourceIP = "" +timeStamp = "" def decodePayload(hexArray): payloadStr = "" @@ -482,9 +484,9 @@ def printLine(): global fullHex if sourceIP != "" and fullHex != []: decodedPayload = decodePayload(fullHex) - z = re.search(r'START(.*)EOF$', decodedPayload) + z = re.search(r'START(.*)EOF_(\d+)', decodedPayload) if z: - print(sourceIP + " " + z.group(1)) + print("Parsed " + timeStamp + " " + sourceIP + " " + z.group(1) + "_" + z.group(2)) fullHex = [] sourceIP = "" @@ -497,6 +499,7 @@ for line in sys.stdin: printLine() elif not re.match(r'^$', line): printLine() + timeStamp = line.split()[0] sourceIP = line.split()[sourceIPOffset] printLine() @@ -545,16 +548,6 @@ func createHostNetworkedPacketSnifferDaemonSet(clientset kubernetes.Interface, n }, }, } - readinessProbe := &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - Exec: &v1.ExecAction{ - Command: []string{ - "echo", - "ready", - }, - }, - }, - } runAsUser := int64(0) securityContext := &v1.SecurityContext{ RunAsUser: &runAsUser, @@ -582,6 +575,12 @@ func createHostNetworkedPacketSnifferDaemonSet(clientset kubernetes.Interface, n Labels: podLabels, }, Spec: corev1.PodSpec{ + Tolerations: []v1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Effect: corev1.TaintEffectNoSchedule, + }, + }, Affinity: &nodeAffinity, HostNetwork: true, Containers: []v1.Container{ @@ -589,7 +588,6 @@ func createHostNetworkedPacketSnifferDaemonSet(clientset kubernetes.Interface, n Name: "tcpdump", Image: networkPacketSnifferImage, Command: podCommand, - ReadinessProbe: readinessProbe, SecurityContext: securityContext, TTY: true, // needed for immediate log propagation Stdin: true, // needed for immediate log propagation @@ -651,31 +649,33 @@ func scanPacketSnifferDaemonSetPodLogs(oc *exutil.CLI, ds *appsv1.DaemonSet, tar scanner := bufio.NewScanner(buf) for scanner.Scan() { logLine := scanner.Text() - if strings.Contains(logLine, searchString) { - // Currently, it is not necessary to discriminate by protocol. - // a log line should look like this for http: - // 10.0.144.5.33226 /bed729aa-4e83-482d-a433-db798e569147 - // a log line should look like this for udp: - // 10.0.144.5.33226 bed729aa-4e83-482d-a433-db798e569147 - // Should it ever be necessary, the targetProtocol to this method (which is currently - // not used) serves this purpose. - framework.Logf("Found hit in log line: %s", logLine) - logLineExploded := strings.Fields(logLine) - if len(logLineExploded) != 2 { - return nil, fmt.Errorf("Unexpected logline content: %s", logLine) - } - ipAddressPortExploded := strings.Split(logLineExploded[0], ".") - if len(ipAddressPortExploded) == 2 { - // ipv6 - ip = ipAddressPortExploded[0] - } else if len(ipAddressPortExploded) == 5 { - // ipv4 - ip = strings.Join(ipAddressPortExploded[:len(ipAddressPortExploded)-1], ".") - } else { - return nil, fmt.Errorf("Unexpected logline content, invalid IP/Port: %s", logLine) - } - matchedIPs[ip]++ + + if !strings.HasPrefix(logLine, "Parsed") || !strings.Contains(logLine, searchString) { + continue + } + // Currently, it is not necessary to discriminate by protocol. + // a log line should look like this for http: + // 10.0.144.5.33226 /bed729aa-4e83-482d-a433-db798e569147 + // a log line should look like this for udp: + // 10.0.144.5.33226 bed729aa-4e83-482d-a433-db798e569147 + // Should it ever be necessary, the targetProtocol to this method (which is currently + // not used) serves this purpose. + framework.Logf("Found hit in log line for node %s: %s", pod.Spec.NodeName, logLine) + logLineExploded := strings.Fields(logLine) + if len(logLineExploded) != 4 { + return nil, fmt.Errorf("Unexpected logline content %s", logLine) + } + ipAddressPortExploded := strings.Split(logLineExploded[2], ".") + if len(ipAddressPortExploded) == 2 { + // ipv6 + ip = ipAddressPortExploded[0] + } else if len(ipAddressPortExploded) == 5 { + // ipv4 + ip = strings.Join(ipAddressPortExploded[:len(ipAddressPortExploded)-1], ".") + } else { + return nil, fmt.Errorf("Unexpected logline content, invalid IP/Port: %s", logLine) } + matchedIPs[ip]++ } } return matchedIPs, nil @@ -1271,26 +1271,28 @@ func sendProbesToHostPort(oc *exutil.CLI, proberPod *v1.Pod, url, targetProtocol request := fmt.Sprintf("http://%s/dial?protocol=%s&host=%s&port=%d&request=%s", url, targetProtocol, targetHost, targetPort, randomIDStr) var wg sync.WaitGroup errChan := make(chan error, iterations) + for i := 0; i < iterations; i++ { // Make sure that we don“t reuse the i variable when passing it to the go func. - i := i + interval := i // Randomize the start time a little bit per go routine. // Max of 250 ms * current iteration counter - n := rand.Intn(250) * i - framework.Logf("Sleeping for %d ms for iteration %d", n, i) + n := rand.Intn(250) * interval + framework.Logf("Sleeping for %d ms for iteration %d", n, interval) wg.Add(1) go func() { defer wg.Done() time.Sleep(time.Duration(n) * time.Millisecond) - output, err := runOcWithRetry(oc.AsAdmin(), "exec", proberPod.Name, "--", "curl", "--max-time", "15", "-s", request) + output, err := runOcWithRetry(oc.AsAdmin(), "exec", proberPod.Name, "--", "curl", "--max-time", "15", "-s", fmt.Sprintf("%s_%d", request, i)) + framework.Logf("Probed with output: %s", output) // Report errors. if err != nil { errChan <- fmt.Errorf("Query failed. Request: %s, Output: %s, Error: %v", request, output, err) } - return }() } wg.Wait() + close(errChan) // Close the channel after all goroutines finish // If the above yielded any errors, then append them to a list and report them. if len(errChan) > 0 { @@ -1729,21 +1731,21 @@ func cloudPrivateIpConfigExists(oc *exutil.CLI, cloudNetworkClientset cloudnetwo } // egressIPStatusHasIP returns if a given ip was found in a given EgressIP object's status field. -func egressIPStatusHasIP(oc *exutil.CLI, egressIPObjectName string, ip string) (bool, error) { +func egressIPStatusHasIP(oc *exutil.CLI, egressIPObjectName string, ip string) (bool, string, error) { eip, err := getEgressIP(oc, egressIPObjectName) if err != nil { if errors.IsNotFound(err) { - return false, nil + return false, "", nil } - return false, fmt.Errorf("Error looking up EgressIP %s, err: %v", egressIPObjectName, err) + return false, "", fmt.Errorf("Error looking up EgressIP %s, err: %v", egressIPObjectName, err) } for _, egressIPStatusItem := range eip.Status.Items { if egressIPStatusItem.EgressIP == ip { - return true, nil + return true, egressIPStatusItem.Node, nil } } - return false, nil + return false, "", nil } // sdnNamespaceAddEgressIP adds EgressIP to netnamespace . diff --git a/test/extended/networking/route_advertisements.go b/test/extended/networking/route_advertisements.go index c5586eebeb1e..d147eac464c8 100644 --- a/test/extended/networking/route_advertisements.go +++ b/test/extended/networking/route_advertisements.go @@ -5,9 +5,11 @@ import ( "context" "encoding/json" "fmt" + "math/rand" "net" "os" "regexp" + "slices" "strings" "time" @@ -48,10 +50,6 @@ const ( frrNamespace = "openshift-frr-k8s" raLabel = "k8s.ovn.org/route-advertisements" - - userDefinedNetworkIPv4Subnet = "203.203.0.0/16" - userDefinedNetworkIPv6Subnet = "2014:100:200::0/60" - cudnName = "udn1" ) type response struct { @@ -71,12 +69,14 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro workerNodesOrdered []corev1.Node workerNodesOrderedNames []string advertisedPodsNodes []string + egressIPNodes []string externalNodeName string targetNamespace string snifferNamespace string cloudType configv1.PlatformType deployName string - routeName string + svcUrl string + cudnName string packetSnifferDaemonSet *v1.DaemonSet podList *corev1.PodList v4PodIPSet map[string]string @@ -153,10 +153,9 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro o.Expect(len(workerNodesOrderedNames)).Should(o.BeNumerically(">", 1)) externalNodeName = workerNodesOrderedNames[0] advertisedPodsNodes = workerNodesOrderedNames[1:] + egressIPNodes = workerNodesOrderedNames[1:] - g.By("Creating a project for the prober pod") - // Create a target project and assign source and target namespace - // to variables for later use. + g.By("Creating a project for the sniffer pod") snifferNamespace = oc.SetupProject() clusterIPFamily = getIPFamilyForCluster(f) @@ -168,8 +167,15 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro os.RemoveAll(tmpDirBGP) }) + g.JustAfterEach(func() { + specReport := g.CurrentSpecReport() + if specReport.Failed() { + gatherDebugInfo(oc, snifferNamespace, targetNamespace, workerNodesOrderedNames) + } + }) + g.Context("[PodNetwork] Advertising the default network [apigroup:user.openshift.io][apigroup:security.openshift.io]", func() { - g.JustBeforeEach(func() { + g.BeforeEach(func() { g.By("Setup packet sniffer at nodes") var err error packetSnifferDaemonSet, err = setupPacketSniffer(oc, clientset, snifferNamespace, advertisedPodsNodes, networkPlugin) @@ -186,9 +192,10 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro } g.By("Deploy the test pods") - deployName, routeName, podList, err = setupTestDeployment(oc, clientset, targetNamespace, advertisedPodsNodes) + deployName, _, podList, err = setupTestDeployment(oc, clientset, targetNamespace, advertisedPodsNodes) o.Expect(err).NotTo(o.HaveOccurred()) o.Expect(len(podList.Items)).To(o.Equal(len(advertisedPodsNodes))) + svcUrl = fmt.Sprintf("%s-0-service.%s.svc.cluster.local:%d", targetNamespace, targetNamespace, serverPort) g.By("Extract test pod IPs") v4PodIPSet, v6PodIPSet = extractPodIPs(podList) @@ -205,12 +212,11 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro if clusterIPFamily == DualStack || clusterIPFamily == IPv4 { g.By("sending to IPv4 external host") - spawnProberSendEgressIPTrafficCheckLogs(oc, snifferNamespace, probePodName, routeName, targetProtocol, v4ExternalIP, serverPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, v4PodIPSet) + spawnProberSendEgressIPTrafficCheckLogs(oc, snifferNamespace, probePodName, svcUrl, targetProtocol, v4ExternalIP, serverPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, v4PodIPSet) } if clusterIPFamily == DualStack || clusterIPFamily == IPv6 { - // [TODO] enable IPv6 test once OCPBUGS-52194 is fixed - // g.By("sending to IPv6 external host") - // spawnProberSendEgressIPTrafficCheckLogs(oc, snifferNamespace, probePodName, routeName, targetProtocol, v6ExternalIP, serverPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, v6PodIPSet) + g.By("sending to IPv6 external host") + spawnProberSendEgressIPTrafficCheckLogs(oc, snifferNamespace, probePodName, svcUrl, targetProtocol, v6ExternalIP, serverPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, v6PodIPSet) } }) @@ -228,16 +234,16 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro } if clusterIPFamily == DualStack || clusterIPFamily == IPv6 { - // [TODO] enable IPv6 test once OCPBUGS-52194 is fixed - // g.By("checking the external host to pod traffic works for IPv6") - // for podIP := range v6PodIPSet { - // checkExternalResponse(oc, proberPod, podIP, v6ExternalIP, serverPort, packetSnifferDaemonSet, targetProtocol) - // } + g.By("checking the external host to pod traffic works for IPv6") + for podIP := range v6PodIPSet { + checkExternalResponse(oc, proberPod, podIP, v6ExternalIP, serverPort, packetSnifferDaemonSet, targetProtocol) + } } }) }) - g.Context("[PodNetwork] Advertising a cluster user defined network [Serial][apigroup:user.openshift.io][apigroup:security.openshift.io]", func() { + g.Context("[PodNetwork] Advertising a cluster user defined network [apigroup:user.openshift.io][apigroup:security.openshift.io]", func() { + var cleanup func() g.BeforeEach(func() { var err error g.By("Setup packet sniffer at nodes") @@ -254,6 +260,8 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro o.Expect(err).NotTo(o.HaveOccurred()) targetNamespace = ns.Name f.Namespace = ns + // use a long cudn (longer than 15 characters) name to work around OCPBUGS-54659 + cudnName = "clusteruserdefinenetwork" + ns.Name g.By("Creating a cluster user defined network") nc := &networkAttachmentConfigParams{ @@ -262,14 +270,18 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro role: "primary", namespace: targetNamespace, } + userDefinedNetworkIPv4Subnet := generateRandomSubnet(IPv4) + userDefinedNetworkIPv6Subnet := generateRandomSubnet(IPv6) + framework.Logf("userDefinedNetworkIPv4Subnet: %s", userDefinedNetworkIPv4Subnet) + framework.Logf("userDefinedNetworkIPv6Subnet: %s", userDefinedNetworkIPv6Subnet) nc.cidr = correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet) cudnManifest := generateClusterUserDefinedNetworkManifest(nc) - cleanup, err := createManifest(targetNamespace, cudnManifest) + cleanup, err = createManifest(targetNamespace, cudnManifest) o.Expect(err).NotTo(o.HaveOccurred()) o.Eventually(clusterUserDefinedNetworkReadyFunc(oc.AdminDynamicClient(), cudnName), 60*time.Second, time.Second).Should(o.Succeed()) g.By("Labeling the UDN for advertisement") - _, err = runOcWithRetry(oc.AsAdmin(), "label", "clusteruserdefinednetworks", "-n", targetNamespace, cudnName, "advertise=true") + _, err = runOcWithRetry(oc.AsAdmin(), "label", "clusteruserdefinednetworks", "-n", targetNamespace, cudnName, "advertise="+cudnName) o.Expect(err).NotTo(o.HaveOccurred()) g.By("Create the route advertisement for UDN") @@ -277,7 +289,7 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro err = applyManifest(targetNamespace, raManifest) o.Expect(err).NotTo(o.HaveOccurred()) - g.By("Ensure the RouteAdvertisements is accepted") + g.By(fmt.Sprintf("Ensure the RouteAdvertisements %s is accepted", cudnName)) waitForRouteAdvertisements(oc, cudnName) g.By("Makes sure the FRR configuration is generated for each node") @@ -288,27 +300,24 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro } g.By("Deploy the test pods") - deployName, routeName, podList, err = setupTestDeployment(oc, clientset, targetNamespace, advertisedPodsNodes) + deployName, _, podList, err = setupTestDeployment(oc, clientset, targetNamespace, advertisedPodsNodes) o.Expect(err).NotTo(o.HaveOccurred()) o.Expect(len(podList.Items)).To(o.Equal(len(advertisedPodsNodes))) + svcUrl = fmt.Sprintf("%s-0-service.%s.svc.cluster.local:%d", targetNamespace, targetNamespace, serverPort) g.By("Extract test pod UDN IPs") v4PodIPSet, v6PodIPSet = extractPodUdnIPs(podList, nc, targetNamespace, clientset) - - g.DeferCleanup(func() { - runOcWithRetry(oc.AsAdmin(), "delete", "deploy", deployName) - runOcWithRetry(oc.AsAdmin(), "delete", "pod", "--all") - runOcWithRetry(oc.AsAdmin(), "delete", "ra", cudnName) - runOcWithRetry(oc.AsAdmin(), "delete", "clusteruserdefinednetwork", cudnName) - cleanup() - }) }) g.AfterEach(func() { + runOcWithRetry(oc.AsAdmin(), "delete", "deploy", deployName) + runOcWithRetry(oc.AsAdmin(), "delete", "pod", "--all") + runOcWithRetry(oc.AsAdmin(), "delete", "ra", cudnName) + runOcWithRetry(oc.AsAdmin(), "delete", "clusteruserdefinednetwork", cudnName) + cleanup() }) g.It("pods should communicate with external host without being SNATed", func() { - g.By("Checking that BGP routes to the PodNetwork are learned by other nodes") g.By("Checking that routes are advertised to each node") for _, nodeName := range workerNodesOrderedNames { verifyLearnedBgpRoutesForNode(oc, nodeName, cudnName) @@ -317,7 +326,6 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro numberOfRequestsToSend := 10 g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and PodIP %v were seen", numberOfRequestsToSend, v4PodIPSet)) - svcUrl := fmt.Sprintf("%s-0-service:%d", targetNamespace, serverPort) if clusterIPFamily == DualStack || clusterIPFamily == IPv4 { g.By("sending to IPv4 external host") spawnProberSendEgressIPTrafficCheckLogs(oc, targetNamespace, probePodName, svcUrl, targetProtocol, v4ExternalIP, serverPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, v4PodIPSet) @@ -349,9 +357,350 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro } }) }) + + g.Context("[EgressIP][apigroup:user.openshift.io][apigroup:security.openshift.io]", func() { + var err error + var egressIPYamlPath, egressIPObjectName string + + g.BeforeEach(func() { + egressIPYamlPath = tmpDirBGP + "/" + egressIPYaml + g.By("Setting the EgressIP nodes as EgressIP assignable") + _, err = runOcWithRetry(oc.AsAdmin(), "create", "configmap", "egressip-test") + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = runOcWithRetry(oc.AsAdmin(), "label", "configmap", "egressip-test", "app=egressip-test") + o.Expect(err).NotTo(o.HaveOccurred()) + for _, node := range egressIPNodes { + _, err = runOcWithRetry(oc.AsAdmin(), "label", "node", node, "k8s.ovn.org/egress-assignable=") + o.Expect(err).NotTo(o.HaveOccurred()) + } + g.By("Setup packet sniffer at nodes") + packetSnifferDaemonSet, err = setupPacketSniffer(oc, clientset, snifferNamespace, egressIPNodes, networkPlugin) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.AfterEach(func() { + g.By("Deleting the EgressIP object if it exists for OVN Kubernetes") + if _, err := os.Stat(egressIPYamlPath); err == nil { + _, _ = runOcWithRetry(oc.AsAdmin(), "delete", "-f", tmpDirBGP+"/"+egressIPYaml) + } + output, _ := runOcWithRetry(oc.AsAdmin(), "get", "egressip", "--no-headers") + if strings.TrimSpace(output) != "No resources found" { + framework.Logf("don't unlabel the nodes if there are still EgressIP objects: %s", output) + return + } + runOcWithRetry(oc.AsAdmin(), "delete", "configmap", "egressip-test") + output, _ = runOcWithRetry(oc.AsAdmin(), "get", "configmap", "--no-headers", "-A", "-l", "app=egressip-test") + if !strings.Contains(output, "NotFound") { + framework.Logf("don't unlabel the nodes if other egress ip test is running: %s", output) + return + } + + g.By("Removing the EgressIP assignable annotation for OVN Kubernetes") + for _, nodeName := range egressIPNodes { + _, _ = runOcWithRetry(oc.AsAdmin(), "label", "node", nodeName, "k8s.ovn.org/egress-assignable-") + } + }) + + g.Context("Advertising egressIP for default network", func() { + g.BeforeEach(func() { + egressIPObjectName = targetNamespace + + g.By("Turn on the BGP advertisement of EgressIPs") + _, err = runOcWithRetry(oc.AsAdmin(), "patch", "ra", "default", "--type=merge", `-p={"spec":{"advertisements":["EgressIP","PodNetwork"]}}`) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Ensure the RouteAdvertisements is accepted") + waitForRouteAdvertisements(oc, "default") + + g.By("Makes sure the FRR configuration is generated for each node") + for _, nodeName := range workerNodesOrderedNames { + frr, err := getGeneratedFrrConfigurationForNode(oc, nodeName, "default") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(frr).NotTo(o.BeNil()) + } + }) + + g.AfterEach(func() { + g.By("Turn off the BGP advertisement of EgressIPs") + _, err := runOcWithRetry(oc.AsAdmin(), "patch", "ra", "default", "--type=merge", `-p={"spec":{"advertisements":["PodNetwork"]}}`) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.DescribeTable("pods should have the assigned EgressIPs and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io]", + func(ipFamily IPFamily, externalIP string) { + if clusterIPFamily != ipFamily && clusterIPFamily != DualStack { + skipper.Skipf("Skipping test for IPFamily: %s", ipFamily) + return + } + g.By("Selecte EgressIP set for the test") + egressIPSet, newEgressIPSet := getEgressIPSet(ipFamily, egressIPNodes) + framework.Logf("egressIPSet: %v", egressIPSet) + + g.By("Deploy the test pods") + deployName, _, podList, err = setupTestDeployment(oc, clientset, targetNamespace, egressIPNodes) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(podList.Items)).To(o.Equal(len(egressIPNodes))) + svcUrl = fmt.Sprintf("%s-0-service.%s.svc.cluster.local:%d", targetNamespace, targetNamespace, serverPort) + + numberOfRequestsToSend := 10 + // Run this twice to make sure that repeated EgressIP creation, update and deletion works. + for i := 0; i < 2; i++ { + g.By("Creating the EgressIP object") + ovnKubernetesCreateEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, targetNamespace, "", egressIPSet) + + g.By("Applying the EgressIP object") + applyEgressIPObject(oc, nil, egressIPYamlPath, targetNamespace, egressIPSet, egressUpdateTimeout) + + g.By("Makes sure the EgressIP is advertised by FRR") + for eip, nodeName := range egressIPSet { + o.Expect(nodeName).ShouldNot(o.BeEmpty()) + o.Eventually(func() bool { + return isRouteToEgressIPPresent(oc, eip, "default", nodeName) + }).WithTimeout(3 * timeOut).WithPolling(interval).Should(o.BeTrue()) + } + + g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, snifferNamespace, probePodName, svcUrl, targetProtocol, externalIP, serverPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) + + g.By("Updating the EgressIP object") + ovnKubernetesCreateEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, targetNamespace, "", newEgressIPSet) + + g.By("Applying the updated EgressIP object") + applyEgressIPObject(oc, nil, egressIPYamlPath, targetNamespace, newEgressIPSet, egressUpdateTimeout) + + g.By("Makes sure the updated EgressIP is advertised by FRR ") + for eip, nodeName := range newEgressIPSet { + o.Expect(nodeName).ShouldNot(o.BeEmpty()) + o.Eventually(func() bool { + return isRouteToEgressIPPresent(oc, eip, "default", nodeName) + }).WithTimeout(3 * timeOut).WithPolling(interval).Should(o.BeTrue()) + } + + g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and updated EgressIPs %v were seen", numberOfRequestsToSend, newEgressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, snifferNamespace, probePodName, svcUrl, targetProtocol, externalIP, serverPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, newEgressIPSet) + + g.By("Deleting the EgressIP object") + // Use cascading foreground deletion to make sure that the EgressIP object and its dependencies are gone. + _, err = runOcWithRetry(oc.AsAdmin(), "delete", "egressip", egressIPObjectName, "--cascade=foreground") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Makes sure the EgressIP is not advertised by FRR") + for eip, nodeName := range newEgressIPSet { + o.Eventually(func() bool { + return isRouteToEgressIPPresent(oc, eip, "default", nodeName) + }).WithTimeout(3 * timeOut).WithPolling(interval).Should(o.BeFalse()) + } + + g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", 0, newEgressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, snifferNamespace, probePodName, svcUrl, targetProtocol, externalIP, serverPort, numberOfRequestsToSend, 0, packetSnifferDaemonSet, newEgressIPSet) + } + }, + g.Entry("IPv4", IPv4, v4ExternalIP), + g.Entry("IPv6", IPv6, v6ExternalIP), + ) + }) + + g.Context("Advertising egressIP for user defined network", func() { + var cleanup func() + g.BeforeEach(func() { + g.By("Create a namespace with UDPN") + ns, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ + "e2e-framework": f.BaseName, + RequiredUDNNamespaceLabel: "", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + err = udnWaitForOpenShift(oc, ns.Name) + o.Expect(err).NotTo(o.HaveOccurred()) + targetNamespace = ns.Name + f.Namespace = ns + egressIPObjectName = targetNamespace + cudnName = ns.Name + + g.By("Creating a cluster user defined network") + nc := &networkAttachmentConfigParams{ + name: cudnName, + topology: "layer3", + role: "primary", + namespace: targetNamespace, + } + userDefinedNetworkIPv4Subnet := generateRandomSubnet(IPv4) + userDefinedNetworkIPv6Subnet := generateRandomSubnet(IPv6) + framework.Logf("userDefinedNetworkIPv4Subnet: %s", userDefinedNetworkIPv4Subnet) + framework.Logf("userDefinedNetworkIPv6Subnet: %s", userDefinedNetworkIPv6Subnet) + nc.cidr = correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet) + cudnManifest := generateClusterUserDefinedNetworkManifest(nc) + cleanup, err = createManifest(targetNamespace, cudnManifest) + + o.Expect(err).NotTo(o.HaveOccurred()) + o.Eventually(clusterUserDefinedNetworkReadyFunc(oc.AdminDynamicClient(), cudnName), 60*time.Second, time.Second).Should(o.Succeed()) + g.By("Labeling the UDN for advertisement") + _, err = runOcWithRetry(oc.AsAdmin(), "label", "clusteruserdefinednetworks", "-n", targetNamespace, cudnName, "advertise="+cudnName) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Create the route advertisement for UDN") + raManifest := newRouteAdvertisementsManifest(cudnName, false, true) + err = applyManifest(targetNamespace, raManifest) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By(fmt.Sprintf("Ensure the RouteAdvertisements %s is accepted", cudnName)) + waitForRouteAdvertisements(oc, cudnName) + }) + + g.AfterEach(func() { + runOcWithRetry(oc.AsAdmin(), "delete", "deploy", deployName) + runOcWithRetry(oc.AsAdmin(), "delete", "ra", cudnName) + runOcWithRetry(oc.AsAdmin(), "delete", "pod", "--all") + runOcWithRetry(oc.AsAdmin(), "delete", "clusteruserdefinednetwork", cudnName) + cleanup() + }) + + g.DescribeTable("UDN pods should have the assigned EgressIPs and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io]", + func(ipFamily IPFamily, externalIP string) { + if clusterIPFamily != ipFamily && clusterIPFamily != DualStack { + skipper.Skipf("Skipping test for IPFamily: %s", ipFamily) + return + } + + g.By("Selecte EgressIP set for the test") + egressIPSet, newEgressIPSet := getEgressIPSet(ipFamily, egressIPNodes) + framework.Logf("egressIPSet: %v", egressIPSet) + + g.By("Deploy the test pods") + deployName, _, podList, err = setupTestDeployment(oc, clientset, targetNamespace, egressIPNodes) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(podList.Items)).To(o.Equal(len(egressIPNodes))) + svcUrl = fmt.Sprintf("%s-0-service.%s.svc.cluster.local:%d", targetNamespace, targetNamespace, serverPort) + + numberOfRequestsToSend := 10 + // Run this twice to make sure that repeated EgressIP creation and deletion works. + for i := 0; i < 2; i++ { + g.By("Creating the EgressIP object") + ovnKubernetesCreateEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, targetNamespace, "", egressIPSet) + + g.By("Applying the EgressIP object") + applyEgressIPObject(oc, nil, egressIPYamlPath, targetNamespace, egressIPSet, egressUpdateTimeout) + + g.By("Makes sure the EgressIP is advertised by FRR") + for eip, nodeName := range egressIPSet { + o.Expect(nodeName).ShouldNot(o.BeEmpty()) + o.Eventually(func() bool { + return isRouteToEgressIPPresent(oc, eip, cudnName, nodeName) + }).WithTimeout(3 * timeOut).WithPolling(interval).Should(o.BeTrue()) + } + + svcUrl := fmt.Sprintf("%s-0-service:%d", targetNamespace, serverPort) + g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, targetNamespace, probePodName, svcUrl, targetProtocol, externalIP, serverPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) + + g.By("Updating the EgressIP object") + ovnKubernetesCreateEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, targetNamespace, "", newEgressIPSet) + + g.By("Applying the updated EgressIP object") + applyEgressIPObject(oc, nil, egressIPYamlPath, targetNamespace, newEgressIPSet, egressUpdateTimeout) + + g.By("Makes sure the updated EgressIP is advertised by FRR ") + for eip, nodeName := range newEgressIPSet { + o.Expect(nodeName).ShouldNot(o.BeEmpty()) + o.Eventually(func() bool { + return isRouteToEgressIPPresent(oc, eip, cudnName, nodeName) + }).WithTimeout(3 * timeOut).WithPolling(interval).Should(o.BeTrue()) + } + + g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and updated EgressIPs %v were seen", numberOfRequestsToSend, newEgressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, targetNamespace, probePodName, svcUrl, targetProtocol, externalIP, serverPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, newEgressIPSet) + + g.By("Deleting the EgressIP object") + // Use cascading foreground deletion to make sure that the EgressIP object and its dependencies are gone. + _, err = runOcWithRetry(oc.AsAdmin(), "delete", "egressip", egressIPObjectName, "--cascade=foreground") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Makes sure the EgressIP is not advertised by FRR") + for eip, nodeName := range newEgressIPSet { + o.Eventually(func() bool { + return isRouteToEgressIPPresent(oc, eip, cudnName, nodeName) + }).WithTimeout(3 * timeOut).WithPolling(interval).Should(o.BeFalse()) + } + + g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", 0, newEgressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, targetNamespace, probePodName, svcUrl, targetProtocol, externalIP, serverPort, numberOfRequestsToSend, 0, packetSnifferDaemonSet, newEgressIPSet) + } + }, + g.Entry("IPv4", IPv4, v4ExternalIP), + g.Entry("IPv6", IPv6, v6ExternalIP), + ) + }) + }) }) }) +func IntnRange(min, max int) int { + return rand.Intn(max-min+1) + min +} + +func generateRandomSubnet(ipFamily IPFamily) string { + var subnet string + switch ipFamily { + case IPv4: + subnet = fmt.Sprintf("203.%d.0.0/16", IntnRange(0, 255)) + case IPv6: + subnet = fmt.Sprintf("2014:100:200:%0x::0/60", IntnRange(0, 255)) + default: + o.Expect(false).To(o.BeTrue()) + } + return subnet +} + +func getEgressIPSet(ipFamily IPFamily, eipNodes []string) (map[string]string, map[string]string) { + egressIPSet := make(map[string]string) + newEgressIPSet := make(map[string]string) + for range eipNodes { + switch ipFamily { + case IPv4: + eip := fmt.Sprintf("192.168.111.%d", IntnRange(30, 254)) + egressIPSet[eip] = "" + neip := fmt.Sprintf("192.168.111.%d", IntnRange(30, 254)) + newEgressIPSet[neip] = "" + case IPv6: + eip := fmt.Sprintf("fd2e:6f44:5dd8:c956::%0x", IntnRange(30, 254)) + egressIPSet[eip] = "" + neip := fmt.Sprintf("fd2e:6f44:5dd8:c956::%0x", IntnRange(30, 254)) + newEgressIPSet[neip] = "" + default: + o.Expect(false).To(o.BeTrue()) + } + } + return egressIPSet, newEgressIPSet +} + +// isRouteToEgressIPPresent checks that routes to the egress IPs are being advertised by FRR. +func isRouteToEgressIPPresent(oc *exutil.CLI, eip, netName, nodeName string) bool { + advertised := false + frr, err := getGeneratedFrrConfigurationForNode(oc, nodeName, netName) + if err != nil && err.Error() == "FRR configuration for node "+nodeName+" not found" { + return advertised + } + o.Expect(err).NotTo(o.HaveOccurred()) + if len(frr.Spec.BGP.Routers) == 0 { + return advertised + } + + // Parse IP to determine if it's IPv4 or IPv6 + ip := net.ParseIP(eip) + o.Expect(ip).NotTo(o.BeNil()) + + var prefix string + if ip.To4() != nil { + prefix = fmt.Sprintf("%s/32", eip) + } else { + prefix = fmt.Sprintf("%s/128", eip) + } + + if slices.Contains(frr.Spec.BGP.Routers[0].Prefixes, prefix) { + advertised = true + } + return advertised +} + // getRouteAdvertisements uses the dynamic admin client to return a pointer to // an existing RouteAdvertisements, or error. func getRouteAdvertisements(oc *exutil.CLI, name string) (*ratypes.RouteAdvertisements, error) { @@ -477,12 +826,14 @@ func getLearnedBgpRoutesByNode(oc *exutil.CLI, nodeName string) (map[string]stri if err != nil { return nil, nil, err } + framework.Logf("BGP v4 routes for node %s: %s", nodeName, out) v4bgpRoutes = parseRoutes(out) out, err = adminExecInPod(oc, frrNamespace, podName, "frr", "ip -6 route show proto bgp") if err != nil { return nil, nil, err } + framework.Logf("BGP v6 routes for node %s: %s", nodeName, out) v6bgpRoutes = parseRoutes(out) return v4bgpRoutes, v6bgpRoutes, nil @@ -515,17 +866,40 @@ func newRouteAdvertisementsManifest(name string, podNetwork, egressip bool) stri if egressip { advertisements = append(advertisements, "EgressIP") } - return fmt.Sprintf(` + if name == "default" { + return fmt.Sprintf(` apiVersion: k8s.ovn.org/v1 kind: RouteAdvertisements metadata: name: %s spec: - advertisements: [%s] - networkSelector: + nodeSelector: {} + frrConfigurationSelector: matchLabels: - advertise: "true" + network: default + advertisements: [%s] + networkSelectors: + - networkSelectionType: DefaultNetwork `, name, strings.Join(advertisements, ",")) + } + return fmt.Sprintf(` +apiVersion: k8s.ovn.org/v1 +kind: RouteAdvertisements +metadata: + name: %s +spec: + nodeSelector: {} + frrConfigurationSelector: + matchLabels: + network: default + advertisements: [%s] + networkSelectors: + - networkSelectionType: ClusterUserDefinedNetworks + clusterUserDefinedNetworkSelector: + networkSelector: + matchLabels: + advertise: "%s" +`, name, strings.Join(advertisements, ","), name) } // verifyLearnedBgpRoutesForNode encapsulates the verification of learned BGP routes for a node. @@ -560,11 +934,11 @@ func verifyLearnedBgpRoutesForNode(oc *exutil.CLI, nodeName string, network stri func verifyExternalRoutes(v4Routes, v6Routes map[string]string) bool { if _, ok := v4Routes[v4ExternalCIDR]; !ok { - framework.Logf("Missing v4 external route %s", v4ExternalCIDR) + framework.Logf("Missing v4 external route %s in %v", v4ExternalCIDR, v4Routes) return false } if _, ok := v6Routes[v6ExternalCIDR]; !ok { - framework.Logf("Missing v6 external route %s", v6ExternalCIDR) + framework.Logf("Missing v6 external route %s in %v", v6ExternalCIDR, v6Routes) return false } return true @@ -584,8 +958,7 @@ func verifyNodeSubnetRoutes(nodeName string, nodeSubnets map[string][]net.IPNet, } else { if _, ok := v6Routes[subnet.String()]; !ok { framework.Logf("Missing v6 route for node %s subnet %s", node, subnet.String()) - // [TODO] enable IPv6 test once OCPBUGS-52194 is fixed - // return false + return false } } } @@ -608,7 +981,7 @@ func checkExternalResponse(oc *exutil.CLI, proberPod *corev1.Pod, podIP, Externa g.By("Making sure that external host IP presents in the sniffer packet log") var lastErr error o.Eventually(func() bool { - output, err := runOcWithRetry(oc.AsAdmin(), "exec", proberPod.Name, "--", "curl", "-s", url) + output, err := runOcWithRetry(oc.AsAdmin(), "exec", proberPod.Name, "--", "curl", "-m 3", "-s", url) if err != nil { lastErr = fmt.Errorf("failed to execute curl command: %v", err) return false @@ -644,7 +1017,7 @@ func checkExternalResponse(oc *exutil.CLI, proberPod *corev1.Pod, podIP, Externa } return true - }, timeOut, interval).Should(o.BeTrue(), func() string { + }, 3*timeOut, interval).Should(o.BeTrue(), func() string { return fmt.Sprintf("Failed to verify external response: %v", lastErr) }) } @@ -734,7 +1107,6 @@ func extractPodUdnIPs(podList *corev1.PodList, nc *networkAttachmentConfigParams var err error v4PodIPSet := make(map[string]string) v6PodIPSet := make(map[string]string) - g.By("Extract test pod UDN IPs") var udnIP string for _, pod := range podList.Items { for i, cidr := range strings.Split(nc.cidr, ",") { @@ -761,3 +1133,139 @@ func extractPodUdnIPs(podList *corev1.PodList, nc *networkAttachmentConfigParams } return v4PodIPSet, v6PodIPSet } + +func runCommandInFrrPods(oc *exutil.CLI, command string) (map[string]string, error) { + results := make(map[string]string) + + // Get all FRR pods + out, err := runOcWithRetry(oc.AsAdmin(), "get", "pods", + "-n", frrNamespace, + "-l", "app=frr-k8s", + "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\t\"}{.spec.nodeName}{\"\\n\"}{end}") + if err != nil { + return nil, fmt.Errorf("failed to get FRR pods: %v", err) + } + + // Process each pod + scanner := bufio.NewScanner(strings.NewReader(out)) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + + // Split line into pod name and node name + parts := strings.Split(line, "\t") + if len(parts) != 2 { + continue + } + podName := parts[0] + nodeName := parts[1] + + // Execute command in pod + output, err := adminExecInPod(oc, frrNamespace, podName, "frr", command) + if err != nil { + framework.Logf("Warning: Command failed in pod %s on node %s: %v", podName, nodeName, err) + continue + } + results[nodeName] = output + } + + return results, nil +} + +func gatherDebugInfo(oc *exutil.CLI, snifferNamespace, targetNamespace string, workerNodesOrderedNames []string) { + if out, err := runOcWithRetry(oc.AsAdmin().WithoutNamespace(), "get", "ra", "-o", "yaml"); err == nil { + framework.Logf("RouteAdvertisements:\n%s", out) + } + if out, err := runOcWithRetry(oc.AsAdmin().WithoutNamespace(), "get", "eip", "-o", "yaml"); err == nil { + framework.Logf("EgressIPs:\n%s", out) + } + if out, err := runOcWithRetry(oc.AsAdmin().WithoutNamespace(), "get", "node", "-l", "k8s.ovn.org/egress-assignable="); err == nil { + framework.Logf("EgressIP assignable nodes:\n%s", out) + } + if out, err := runOcWithRetry(oc.AsAdmin().WithoutNamespace(), "get", "clusteruserdefinednetwork", "-o", "yaml"); err == nil { + framework.Logf("ClusterUserDefinedNetworks:\n%s", out) + } + if out, err := runOcWithRetry(oc.AsAdmin().WithoutNamespace(), "get", "pod", "-n", targetNamespace, "-o", "yaml"); err == nil { + framework.Logf(" %s:\n%s", targetNamespace, out) + } + if out, err := runOcWithRetry(oc.AsAdmin().WithoutNamespace(), "get", "ds", "-n", snifferNamespace, "-o", "yaml"); err == nil { + framework.Logf("DaemonSets in namespace %s:\n%s", snifferNamespace, out) + } + if out, err := runOcWithRetry(oc.AsAdmin().WithoutNamespace(), "get", "pod", "-n", snifferNamespace, "-o", "yaml"); err == nil { + framework.Logf("Pods in namespace %s:\n%s", snifferNamespace, out) + } + if out, err := runOcWithRetry(oc.AsAdmin().WithoutNamespace(), "get", "frrconfiguration", "-n", "openshift-frr-k8s", "-o", "yaml"); err == nil { + framework.Logf("FrrConfiguration:\n%s", out) + } + if out, err := runOcWithRetry(oc.AsAdmin().WithoutNamespace(), "get", "frrnodestates", "-o", "yaml"); err == nil { + framework.Logf("FrrNodeStates:\n%s", out) + } + + // FRR debugging information + framework.Logf("\n=== FRR Debugging Information ===") + + if results, err := runCommandInFrrPods(oc, "vtysh -c 'show ip route'"); err == nil { + framework.Logf("\nBGP IPv4 route:") + for node, output := range results { + framework.Logf("Node %s:\n%s", node, output) + } + } + + if results, err := runCommandInFrrPods(oc, "vtysh -c 'show ipv6 route'"); err == nil { + framework.Logf("\nBGP IPv6 route:") + for node, output := range results { + framework.Logf("Node %s:\n%s", node, output) + } + } + + if results, err := runCommandInFrrPods(oc, "vtysh -c 'show ip bgp summary'"); err == nil { + framework.Logf("\nBGP IPv4 Summary:") + for node, output := range results { + framework.Logf("Node %s:\n%s", node, output) + } + } + + if results, err := runCommandInFrrPods(oc, "vtysh -c 'show ip bgp ipv6 summary'"); err == nil { + framework.Logf("\nBGP IPv6 Summary:") + for node, output := range results { + framework.Logf("Node %s:\n%s", node, output) + } + } + + if results, err := runCommandInFrrPods(oc, "vtysh -c 'show bgp ipv4'"); err == nil { + framework.Logf("\nBGP IPv4 Routes:") + for node, output := range results { + framework.Logf("Node %s:\n%s", node, output) + } + } + + if results, err := runCommandInFrrPods(oc, "vtysh -c 'show bgp ipv6'"); err == nil { + framework.Logf("\nBGP IPv6 Routes:") + for node, output := range results { + framework.Logf("Node %s:\n%s", node, output) + } + } + + if results, err := runCommandInFrrPods(oc, "vtysh -c 'show bgp neighbor'"); err == nil { + framework.Logf("\nBGP Neighbors:") + for node, output := range results { + framework.Logf("Node %s:\n%s", node, output) + } + } + + if results, err := runCommandInFrrPods(oc, "vtysh -c 'show bfd peer'"); err == nil { + framework.Logf("\nBFD Peers:") + for node, output := range results { + framework.Logf("Node %s:\n%s", node, output) + } + } + + if results, err := runCommandInFrrPods(oc, "vtysh -c 'show running-config'"); err == nil { + framework.Logf("\nFRR Running Config:") + for node, output := range results { + framework.Logf("Node %s:\n%s", node, output) + } + } +} diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index 17523428b3fe..41c1c51c28e6 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -1759,9 +1759,17 @@ var Annotations = map[string]string{ "[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][Feature:Layer2LiveMigration] Kubevirt Virtual Machines when using openshift ovn-kubernetes with user defined networks and persistent ips configured created using [OCPFeatureGate:NetworkSegmentation] UserDefinedNetwork [Suite:openshift/network/virtualization] should keep ip when the VMI attached to a secondary UDN is migrated between nodes": "", - "[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [PodNetwork] Advertising a cluster user defined network [Serial][apigroup:user.openshift.io][apigroup:security.openshift.io] External host should be able to query route advertised pods by the pod IP": " [Suite:openshift/conformance/serial]", + "[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [EgressIP][apigroup:user.openshift.io][apigroup:security.openshift.io] Advertising egressIP for default network pods should have the assigned EgressIPs and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io] IPv4": " [Suite:openshift/conformance/parallel]", - "[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [PodNetwork] Advertising a cluster user defined network [Serial][apigroup:user.openshift.io][apigroup:security.openshift.io] pods should communicate with external host without being SNATed": " [Suite:openshift/conformance/serial]", + "[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [EgressIP][apigroup:user.openshift.io][apigroup:security.openshift.io] Advertising egressIP for default network pods should have the assigned EgressIPs and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io] IPv6": " [Suite:openshift/conformance/parallel]", + + "[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [EgressIP][apigroup:user.openshift.io][apigroup:security.openshift.io] Advertising egressIP for user defined network UDN pods should have the assigned EgressIPs and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io] IPv4": " [Suite:openshift/conformance/parallel]", + + "[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [EgressIP][apigroup:user.openshift.io][apigroup:security.openshift.io] Advertising egressIP for user defined network UDN pods should have the assigned EgressIPs and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io] IPv6": " [Suite:openshift/conformance/parallel]", + + "[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [PodNetwork] Advertising a cluster user defined network [apigroup:user.openshift.io][apigroup:security.openshift.io] External host should be able to query route advertised pods by the pod IP": " [Suite:openshift/conformance/parallel]", + + "[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [PodNetwork] Advertising a cluster user defined network [apigroup:user.openshift.io][apigroup:security.openshift.io] pods should communicate with external host without being SNATed": " [Suite:openshift/conformance/parallel]", "[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [PodNetwork] Advertising the default network [apigroup:user.openshift.io][apigroup:security.openshift.io] External host should be able to query route advertised pods by the pod IP": " [Suite:openshift/conformance/parallel]", diff --git a/zz_generated.manifests/test-reporting.yaml b/zz_generated.manifests/test-reporting.yaml index 2afb933b8b97..7d6ddfcbfe54 100644 --- a/zz_generated.manifests/test-reporting.yaml +++ b/zz_generated.manifests/test-reporting.yaml @@ -554,13 +554,33 @@ spec: Invalid PIS leads to degraded MCN in a standard Pool [apigroup:machineconfiguration.openshift.io]' - featureGate: RouteAdvertisements tests: + - testName: '[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] + when using openshift ovn-kubernetes [EgressIP][apigroup:user.openshift.io][apigroup:security.openshift.io] + Advertising egressIP for default network pods should have the assigned EgressIPs + and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io] + IPv4' + - testName: '[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] + when using openshift ovn-kubernetes [EgressIP][apigroup:user.openshift.io][apigroup:security.openshift.io] + Advertising egressIP for default network pods should have the assigned EgressIPs + and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io] + IPv6' + - testName: '[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] + when using openshift ovn-kubernetes [EgressIP][apigroup:user.openshift.io][apigroup:security.openshift.io] + Advertising egressIP for user defined network UDN pods should have the assigned + EgressIPs and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io] + IPv4' + - testName: '[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] + when using openshift ovn-kubernetes [EgressIP][apigroup:user.openshift.io][apigroup:security.openshift.io] + Advertising egressIP for user defined network UDN pods should have the assigned + EgressIPs and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io] + IPv6' - testName: '[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [PodNetwork] Advertising a cluster user - defined network [Serial][apigroup:user.openshift.io][apigroup:security.openshift.io] + defined network [apigroup:user.openshift.io][apigroup:security.openshift.io] External host should be able to query route advertised pods by the pod IP' - testName: '[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [PodNetwork] Advertising a cluster user - defined network [Serial][apigroup:user.openshift.io][apigroup:security.openshift.io] + defined network [apigroup:user.openshift.io][apigroup:security.openshift.io] pods should communicate with external host without being SNATed' - testName: '[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:RouteAdvertisements][apigroup:operator.openshift.io] when using openshift ovn-kubernetes [PodNetwork] Advertising the default network