diff --git a/test/extended/router/dualstack.go b/test/extended/router/dualstack.go new file mode 100644 index 000000000000..74b2721b5ac1 --- /dev/null +++ b/test/extended/router/dualstack.go @@ -0,0 +1,350 @@ +package router + +import ( + "context" + "fmt" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" + routev1 "github.com/openshift/api/route/v1" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + admissionapi "k8s.io/pod-security-admission/api" + utilpointer "k8s.io/utils/pointer" + + "github.com/openshift/origin/test/extended/router/shard" + exutil "github.com/openshift/origin/test/extended/util" + "github.com/openshift/origin/test/extended/util/image" +) + +var _ = g.Describe("[sig-network-edge][OCPFeatureGate:AWSDualStackInstall][Feature:Router][apigroup:route.openshift.io][apigroup:operator.openshift.io][apigroup:config.openshift.io]", func() { + defer g.GinkgoRecover() + + var oc = exutil.NewCLIWithPodSecurityLevel("router-dualstack", admissionapi.LevelBaseline) + + var baseDomain string + + g.BeforeEach(func() { + requireAWSDualStack(context.Background(), oc) + + defaultDomain, err := getDefaultIngressClusterDomainName(oc, time.Minute) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to find default domain name") + baseDomain = strings.TrimPrefix(defaultDomain, "apps.") + }) + + g.AfterEach(func() { + if g.CurrentSpecReport().Failed() { + exutil.DumpPodLogsStartingWithInNamespace("router-"+oc.Namespace(), "openshift-ingress", oc.AsAdmin()) + } + }) + + g.It("should be reachable via IPv4 and IPv6 through a dual-stack ingress controller", func() { + ctx := context.Background() + + ns := oc.KubeFramework().Namespace.Name + shardFQDN := "nlb." + baseDomain + + // Deploy the shard first so DNS and LB can provision while we set up the backend. + g.By("Deploying a new router shard with NLB") + shardIngressCtrl, err := shard.DeployNewRouterShard(oc, 10*time.Minute, shard.Config{ + Domain: shardFQDN, + Type: oc.Namespace(), + LoadBalancer: &operatorv1.LoadBalancerStrategy{ + Scope: operatorv1.ExternalLoadBalancer, + ProviderParameters: &operatorv1.ProviderLoadBalancerParameters{ + Type: operatorv1.AWSLoadBalancerProvider, + AWS: &operatorv1.AWSLoadBalancerParameters{ + Type: operatorv1.AWSNetworkLoadBalancer, + }, + }, + }, + }) + defer func() { + if shardIngressCtrl != nil { + if err := oc.AdminOperatorClient().OperatorV1().IngressControllers(shardIngressCtrl.Namespace).Delete(ctx, shardIngressCtrl.Name, metav1.DeleteOptions{}); err != nil { + e2e.Logf("deleting ingress controller failed: %v\n", err) + } + } + }() + o.Expect(err).NotTo(o.HaveOccurred(), "new router shard did not rollout") + + g.By("Disabling client IP preservation on the NLB target group to avoid hairpin issues (OCPBUGS-63219)") + routerSvcName := "router-" + shardIngressCtrl.Name + err = oc.AsAdmin().Run("annotate").Args("service", "-n", "openshift-ingress", routerSvcName, + "service.beta.kubernetes.io/aws-load-balancer-target-group-attributes=preserve_client_ip.enabled=false").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Labelling the namespace for the shard") + err = oc.AsAdmin().Run("label").Args("namespace", oc.Namespace(), "type="+oc.Namespace()).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating backend service and pod") + createBackendServiceAndPod(ctx, oc, ns, "dualstack-backend") + + g.By("Creating an edge-terminated route") + routeHost := "dualstack-test." + shardFQDN + createEdgeRoute(ctx, oc, ns, "dualstack-route", routeHost, "dualstack-backend") + + g.By("Waiting for the route to be admitted") + waitForRouteAdmitted(ctx, oc, ns, "dualstack-route", routeHost, 5*time.Minute) + + g.By("Creating exec pod for curl tests") + execPod := exutil.CreateExecPodOrFail(oc.AdminKubeClient(), ns, "execpod") + defer func() { + oc.AdminKubeClient().CoreV1().Pods(ns).Delete(ctx, execPod.Name, *metav1.NewDeleteOptions(1)) + }() + + g.By("Waiting for DNS resolution of the route host") + err = waitForDNSResolution(ctx, ns, execPod.Name, routeHost, 10*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred(), "DNS resolution failed") + + g.By("Verifying route is reachable over IPv4") + err = waitForRouteResponse(ctx, ns, execPod.Name, routeHost, "-4", 5*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred(), "route not reachable over IPv4") + + g.By("Verifying route is reachable over IPv6") + err = waitForRouteResponse(ctx, ns, execPod.Name, routeHost, "-6", 5*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred(), "route not reachable over IPv6") + }) + + g.It("should be reachable via IPv4 through a Classic LB ingress controller on a dual-stack cluster", func() { + ctx := context.Background() + + ns := oc.KubeFramework().Namespace.Name + shardFQDN := "clb." + baseDomain + + // Deploy the shard first so DNS and LB can provision while we set up the backend. + g.By("Deploying a new router shard with Classic LB") + shardIngressCtrl, err := shard.DeployNewRouterShard(oc, 10*time.Minute, shard.Config{ + Domain: shardFQDN, + Type: oc.Namespace(), + LoadBalancer: &operatorv1.LoadBalancerStrategy{ + Scope: operatorv1.ExternalLoadBalancer, + ProviderParameters: &operatorv1.ProviderLoadBalancerParameters{ + Type: operatorv1.AWSLoadBalancerProvider, + AWS: &operatorv1.AWSLoadBalancerParameters{ + Type: operatorv1.AWSClassicLoadBalancer, + }, + }, + }, + }) + defer func() { + if shardIngressCtrl != nil { + if err := oc.AdminOperatorClient().OperatorV1().IngressControllers(shardIngressCtrl.Namespace).Delete(ctx, shardIngressCtrl.Name, metav1.DeleteOptions{}); err != nil { + e2e.Logf("deleting ingress controller failed: %v\n", err) + } + } + }() + o.Expect(err).NotTo(o.HaveOccurred(), "new router shard did not rollout") + + g.By("Labelling the namespace for the shard") + err = oc.AsAdmin().Run("label").Args("namespace", oc.Namespace(), "type="+oc.Namespace()).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating backend service and pod") + createBackendServiceAndPod(ctx, oc, ns, "classic-backend") + + g.By("Creating an edge-terminated route") + routeHost := "classic-test." + shardFQDN + createEdgeRoute(ctx, oc, ns, "classic-route", routeHost, "classic-backend") + + g.By("Waiting for the route to be admitted") + waitForRouteAdmitted(ctx, oc, ns, "classic-route", routeHost, 5*time.Minute) + + g.By("Creating exec pod for curl tests") + execPod := exutil.CreateExecPodOrFail(oc.AdminKubeClient(), ns, "execpod") + defer func() { + oc.AdminKubeClient().CoreV1().Pods(ns).Delete(ctx, execPod.Name, *metav1.NewDeleteOptions(1)) + }() + + g.By("Waiting for DNS resolution of the route host") + err = waitForDNSResolution(ctx, ns, execPod.Name, routeHost, 10*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred(), "DNS resolution failed") + + g.By("Verifying route is reachable over IPv4") + err = waitForRouteResponse(ctx, ns, execPod.Name, routeHost, "-4", 5*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred(), "route not reachable over IPv4") + }) +}) + +func requireAWSDualStack(ctx context.Context, oc *exutil.CLI) { + infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get infrastructure CR") + + if infra.Status.PlatformStatus == nil || infra.Status.PlatformStatus.Type != configv1.AWSPlatformType { + g.Skip("Test requires AWS platform") + } + if infra.Status.PlatformStatus.AWS == nil { + g.Skip("AWS platform status is not set") + } + ipFamily := infra.Status.PlatformStatus.AWS.IPFamily + if ipFamily != configv1.DualStackIPv4Primary && ipFamily != configv1.DualStackIPv6Primary { + g.Skip(fmt.Sprintf("Test requires DualStack IPFamily, got %q", ipFamily)) + } +} + +func createBackendServiceAndPod(ctx context.Context, oc *exutil.CLI, ns, name string) { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{"app": name}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"app": name}, + IPFamilyPolicy: func() *corev1.IPFamilyPolicy { + p := corev1.IPFamilyPolicyPreferDualStack + return &p + }(), + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 8080, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt(8080), + }, + }, + }, + } + _, err := oc.AdminKubeClient().CoreV1().Services(ns).Create(ctx, service, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: utilpointer.Int64(1), + Containers: []corev1.Container{ + { + Name: "server", + Image: image.ShellImage(), + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/bin/bash", "-c", `while true; do +printf "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nContent-Type: text/plain\r\n\r\nOK" | ncat -l 8080 --send-only || true +done`}, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 8080, + Name: "http", + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + } + _, err = oc.AdminKubeClient().CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + e2e.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(ctx, oc.KubeClient(), name, ns), "backend pod not running") +} + +func createEdgeRoute(ctx context.Context, oc *exutil.CLI, ns, name, host, serviceName string) { + route := routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "type": oc.Namespace(), + }, + }, + Spec: routev1.RouteSpec{ + Host: host, + Port: &routev1.RoutePort{ + TargetPort: intstr.FromInt(8080), + }, + TLS: &routev1.TLSConfig{ + Termination: routev1.TLSTerminationEdge, + InsecureEdgeTerminationPolicy: routev1.InsecureEdgeTerminationPolicyRedirect, + }, + To: routev1.RouteTargetReference{ + Kind: "Service", + Name: serviceName, + Weight: utilpointer.Int32(100), + }, + WildcardPolicy: routev1.WildcardPolicyNone, + }, + } + _, err := oc.RouteClient().RouteV1().Routes(ns).Create(ctx, &route, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func waitForRouteAdmitted(ctx context.Context, oc *exutil.CLI, ns, name, host string, timeout time.Duration) { + err := wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + r, err := oc.RouteClient().RouteV1().Routes(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + e2e.Logf("failed to get route: %v, retrying...", err) + return false, nil + } + for _, ingress := range r.Status.Ingress { + if ingress.Host == host { + for _, condition := range ingress.Conditions { + if condition.Type == routev1.RouteAdmitted && condition.Status == corev1.ConditionTrue { + return true, nil + } + } + } + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "route was not admitted") +} + +func waitForDNSResolution(ctx context.Context, ns, execPodName, host string, timeout time.Duration) error { + cmd := fmt.Sprintf("getent hosts %s", host) + var lastOutput string + err := wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + output, err := e2eoutput.RunHostCmd(ns, execPodName, cmd) + lastOutput = output + if err != nil { + return false, nil + } + e2e.Logf("DNS resolution for %s:\n%s", host, strings.TrimSpace(output)) + return true, nil + }) + if err != nil { + return fmt.Errorf("DNS resolution for %s timed out, last output: %s", host, lastOutput) + } + return nil +} + +func waitForRouteResponse(ctx context.Context, ns, execPodName, host, ipFlag string, timeout time.Duration) error { + curlCmd := fmt.Sprintf("curl %s -k -v -m 10 --connect-timeout 5 -o /dev/null https://%s 2>&1", ipFlag, host) + var lastOutput string + consecutiveSuccesses := 0 + requiredSuccesses := 3 + err := wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + output, err := e2eoutput.RunHostCmd(ns, execPodName, curlCmd) + lastOutput = output + if err != nil { + consecutiveSuccesses = 0 + return false, nil + } + if strings.Contains(output, "< HTTP/1.1 200") || strings.Contains(output, "< HTTP/2 200") { + consecutiveSuccesses++ + e2e.Logf("curl %s %s: success (%d/%d)", ipFlag, host, consecutiveSuccesses, requiredSuccesses) + if consecutiveSuccesses >= requiredSuccesses { + e2e.Logf("curl %s %s:\n%s", ipFlag, host, output) + return true, nil + } + return false, nil + } + consecutiveSuccesses = 0 + return false, nil + }) + if err != nil { + return fmt.Errorf("curl %s to %s timed out, last output:\n%s", ipFlag, host, lastOutput) + } + return nil +} diff --git a/test/extended/router/shard/shard.go b/test/extended/router/shard/shard.go index f5dbe462a7dd..7e3735c4e0c0 100644 --- a/test/extended/router/shard/shard.go +++ b/test/extended/router/shard/shard.go @@ -21,6 +21,10 @@ type Config struct { // Type is the matchSelector Type string + + // LoadBalancer optionally specifies LoadBalancerStrategy parameters. + // If nil, the default LoadBalancer configuration is used. + LoadBalancer *operatorv1.LoadBalancerStrategy } var ingressControllerNonDefaultAvailableConditions = []operatorv1.OperatorCondition{ @@ -43,7 +47,8 @@ func DeployNewRouterShard(oc *exutil.CLI, timeout time.Duration, cfg Config) (*o Replicas: utilpointer.Int32(1), Domain: cfg.Domain, EndpointPublishingStrategy: &operatorv1.EndpointPublishingStrategy{ - Type: operatorv1.LoadBalancerServiceStrategyType, + Type: operatorv1.LoadBalancerServiceStrategyType, + LoadBalancer: cfg.LoadBalancer, }, NodePlacement: &operatorv1.NodePlacement{ NodeSelector: &metav1.LabelSelector{