diff --git a/test/e2e/all_test.go b/test/e2e/all_test.go index 2f70de3d7e..01c23a5d77 100644 --- a/test/e2e/all_test.go +++ b/test/e2e/all_test.go @@ -20,7 +20,6 @@ func TestAll(t *testing.T) { // parallel subtests complete. Each "parallel" test must // invoke t.Parallel(). t.Run("parallel", func(t *testing.T) { - t.Run("TestAWSELBConnectionIdleTimeout", TestAWSELBConnectionIdleTimeout) t.Run("TestClientTLS", TestClientTLS) t.Run("TestContainerLogging", TestContainerLogging) t.Run("TestCustomErrorpages", TestCustomErrorpages) @@ -77,6 +76,7 @@ func TestAll(t *testing.T) { }) t.Run("serial", func(t *testing.T) { + t.Run("TestAWSELBConnectionIdleTimeout", TestAWSELBConnectionIdleTimeout) t.Run("TestDefaultIngressControllerSteadyConditions", TestDefaultIngressControllerSteadyConditions) t.Run("TestClusterOperatorStatusRelatedObjects", TestClusterOperatorStatusRelatedObjects) t.Run("TestConfigurableRouteNoConsumingUserNoRBAC", TestConfigurableRouteNoConsumingUserNoRBAC) diff --git a/test/e2e/operator_test.go b/test/e2e/operator_test.go index eec3d18f4f..611f04c3e4 100644 --- a/test/e2e/operator_test.go +++ b/test/e2e/operator_test.go @@ -13,6 +13,8 @@ import ( "fmt" "io" "io/ioutil" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" "net" "net/http" "os" @@ -2560,7 +2562,7 @@ func TestNetworkLoadBalancer(t *testing.T) { // TestAWSELBConnectionIdleTimeout verifies that the AWS ELB connection-idle // timeout works as expected. func TestAWSELBConnectionIdleTimeout(t *testing.T) { - t.Parallel() + //t.Parallel() if infraConfig.Status.PlatformStatus == nil { t.Skip("test skipped on nil platform") } @@ -2642,14 +2644,237 @@ func TestAWSELBConnectionIdleTimeout(t *testing.T) { t.Fatalf("expected %s=%s, found %s=%s", key, expected, key, v) } + // We need an image that we can use for test clients. The router image + // has Curl, so we can use that image. + deployment := &appsv1.Deployment{} + deploymentName := controller.RouterDeploymentName(ic) + if err := kclient.Get(context.TODO(), deploymentName, deployment); err != nil { + t.Fatalf("failed to get deployment %q: %v", deploymentName, err) + } + + // We need a client-go client in order to execute commands in the client + // pod. + kubeConfig, err := config.GetConfig() + if err != nil { + t.Fatalf("failed to get kube config: %v", err) + } + cl, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + t.Fatalf("failed to create kube client: %v", err) + } + + // Create a test client pod with the client certificates. We will exec + // curl commands in this pod to perform the tests. + podName := "client-local-netlookup" + image := deployment.Spec.Template.Spec.Containers[0].Image + clientPod := buildExecPod(podName, "openshift-ingress", image) + clientPodName := types.NamespacedName{ + Name: clientPod.Name, + Namespace: clientPod.Namespace, + } + if err := kclient.Create(context.TODO(), clientPod); err != nil { + t.Fatalf("failed to create pod %q: %v", clientPodName, err) + } + defer func() { + if err := kclient.Delete(context.TODO(), clientPod); err != nil { + if apierrors.IsNotFound(err) { + return + } + t.Fatalf("failed to delete pod %q: %v", clientPodName, err) + } + }() + + err = wait.PollImmediate(2*time.Second, 3*time.Minute, func() (bool, error) { + if err := kclient.Get(context.TODO(), clientPodName, clientPod); err != nil { + t.Logf("failed to get client pod %q: %v", clientPodName, err) + return false, nil + } + for _, cond := range clientPod.Status.Conditions { + if cond.Type == corev1.PodReady { + return cond.Status == corev1.ConditionTrue, nil + } + } + return false, nil + }) + if err != nil { + t.Fatalf("timed out waiting for pod %q to become ready: %v", clientPodName, err) + } + + // Create a test client pod with the client certificates. We will exec + // curl commands in this pod to perform the tests. + podName = "client-nodedns-netlookup" + clientPodNodeDNS := buildExecPodNodeDNS(podName, "openshift-ingress", image) + clientPodNodeDNSName := types.NamespacedName{ + Name: clientPodNodeDNS.Name, + Namespace: clientPodNodeDNS.Namespace, + } + if err := kclient.Create(context.TODO(), clientPodNodeDNS); err != nil { + t.Fatalf("failed to create pod %q: %v", clientPodNodeDNSName, err) + } + defer func() { + if err := kclient.Delete(context.TODO(), clientPodNodeDNS); err != nil { + if apierrors.IsNotFound(err) { + return + } + t.Fatalf("failed to delete pod %q: %v", clientPodNodeDNSName, err) + } + }() + + err = wait.PollImmediate(2*time.Second, 3*time.Minute, func() (bool, error) { + if err := kclient.Get(context.TODO(), clientPodNodeDNSName, clientPodNodeDNS); err != nil { + t.Logf("failed to get client pod %q: %v", clientPodNodeDNSName, err) + return false, nil + } + for _, cond := range clientPodNodeDNS.Status.Conditions { + if cond.Type == corev1.PodReady { + return cond.Status == corev1.ConditionTrue, nil + } + } + return false, nil + }) + if err != nil { + t.Fatalf("timed out waiting for pod %q to become ready: %v", clientPodNodeDNSName, err) + } + + clusterLocalNetLookup := func(host string, resolver string, useTcp bool) error { + req := cl.CoreV1().RESTClient().Post().Resource("pods"). + Namespace(clientPod.Namespace).Name(clientPod.Name). + SubResource("exec"). + Param("container", clientPod.Spec.Containers[0].Name) + cmd := []string{ + "/bin/dig", "+noall", "+comments", "+answer", "+stats", host, + } + if resolver != "" { + cmd = append(cmd, "@"+resolver) + } + if useTcp { + cmd = append(cmd, "+tcp") + } + req.VersionedParams(&corev1.PodExecOptions{ + Command: cmd, + Stdout: true, + Stderr: true, + }, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(kubeConfig, "POST", req.URL()) + if err != nil { + return err + } + var stdout, stderr bytes.Buffer + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + stdoutStr := stdout.String() + t.Logf("command: %s\nstdout:\n%s\nstderr:\n%s\n", + strings.Join(cmd, " "), stdoutStr, stderr.String()) + if err != nil { + return err + } + return nil + } + + clusterLocalNetLookupNodeDNS := func(host string, resolver string, useTcp bool) error { + req := cl.CoreV1().RESTClient().Post().Resource("pods"). + Namespace(clientPodNodeDNS.Namespace).Name(clientPodNodeDNS.Name). + SubResource("exec"). + Param("container", clientPodNodeDNS.Spec.Containers[0].Name) + cmd := []string{ + "/bin/dig", "+noall", "+comments", "+answer", "+stats", host, + } + if resolver != "" { + cmd = append(cmd, "@"+resolver) + } + if useTcp { + cmd = append(cmd, "+tcp") + } + req.VersionedParams(&corev1.PodExecOptions{ + Command: cmd, + Stdout: true, + Stderr: true, + }, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(kubeConfig, "POST", req.URL()) + if err != nil { + return err + } + var stdout, stderr bytes.Buffer + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + stdoutStr := stdout.String() + t.Logf("command [nodedns]: %s\nstdout:\n%s\nstderr:\n%s\n", + strings.Join(cmd, " "), stdoutStr, stderr.String()) + if err != nil { + return err + } + return nil + } + // Make sure we can resolve the route's host name. It may take some // time for the ingresscontroller's wildcard DNS record to propagate. - if err := wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) { - _, err := net.LookupIP(route.Spec.Host) + if err := wait.PollImmediate(5*time.Second, 20*time.Minute, func() (bool, error) { + // Resolve directly via 8.8.8.8, not via CoreDNS + r := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{ + Timeout: time.Millisecond * time.Duration(20000), + } + return d.DialContext(ctx, network, "8.8.8.8:53") + }, + } + now := time.Now() + ips_direct, errDirect := r.LookupHost(context.Background(), route.Spec.Host) + + if errDirect != nil { + t.Logf("[@8.8.8.8] took %s failed to resolve: %v", time.Now().Sub(now), errDirect) + } else { + t.Logf("[@8.8.8.8] took %s resolved %s to %v", time.Now().Sub(now), route.Spec.Host, ips_direct) + } + + err = clusterLocalNetLookup(route.Spec.Host, "", false) + if err != nil { + t.Error(err) + } + + err = clusterLocalNetLookupNodeDNS(route.Spec.Host, "", false) + if err != nil { + t.Error(err) + } + + lbService := &corev1.Service{} + if err := kclient.Get(context.TODO(), controller.LoadBalancerServiceName(ic), lbService); err != nil { + t.Fatalf("failed to get LoadBalancer service: %v", err) + } + + ips, err := net.LookupIP(lbService.Status.LoadBalancer.Ingress[0].Hostname) + if err != nil { + t.Logf("[loadbalancer_hostname] failed to resolve %s", lbService.Status.LoadBalancer.Ingress[0].Hostname) + } else { + t.Logf("[loadbalancer_hostname] resolved %s to %v", lbService.Status.LoadBalancer.Ingress[0].Hostname, ips) + } + + testHostname := fmt.Sprintf("test.%s", ic.Spec.Domain) + ips, err = net.LookupIP(testHostname) + if err != nil { + t.Logf("[test_hostname] failed to resolve %s", testHostname) + } else { + t.Logf("[test_hostname] resolved %s to %v", testHostname, ips) + } + + ips, err = net.LookupIP(route.Spec.Host) + if err != nil { + t.Logf("[route-host] failed to resolve %s", route.Spec.Host) + } else { + t.Logf("[route-host] resolved %s to %v", route.Spec.Host, ips) + } + + ips, err = net.LookupIP(route.Spec.Host) if err != nil { t.Log(err) return false, nil } + t.Logf("resolved %s to %v", route.Spec.Host, ips) return true, nil }); err != nil { @@ -2757,7 +2982,7 @@ func TestAWSELBConnectionIdleTimeout(t *testing.T) { // Open a connection to the route, send a request, and verify that a // response is received. Use a polling loop because the ELB may need // time to assume the new idle timeout value. - if err := wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) { + if err := wait.PollImmediate(5*time.Second, 10*time.Minute, func() (bool, error) { start := time.Now() response, err := client.Do(request) if err != nil { @@ -2780,6 +3005,70 @@ func TestAWSELBConnectionIdleTimeout(t *testing.T) { }); err != nil { t.Fatalf("failed to observe expected condition: %v", err) } + t.Logf("Everything resolved, now just testing DNS resolution for 10 minutes") + // Make sure we can resolve the route's host name. It may take some + // time for the ingresscontroller's wildcard DNS record to propagate. + wait.PollImmediate(5*time.Second, 10*time.Minute, func() (bool, error) { + // Resolve directly via 8.8.8.8, not via CoreDNS + r := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{ + Timeout: time.Millisecond * time.Duration(20000), + } + return d.DialContext(ctx, network, "8.8.8.8:53") + }, + } + now := time.Now() + ips_direct, errDirect := r.LookupHost(context.Background(), route.Spec.Host) + + if errDirect != nil { + t.Logf("[@8.8.8.8] took %s failed to resolve: %v", time.Now().Sub(now), errDirect) + } else { + t.Logf("[@8.8.8.8] took %s resolved %s to %v", time.Now().Sub(now), route.Spec.Host, ips_direct) + } + + err = clusterLocalNetLookup(route.Spec.Host, "", false) + if err != nil { + t.Error(err) + } + + err = clusterLocalNetLookupNodeDNS(route.Spec.Host, "", false) + if err != nil { + t.Error(err) + } + + lbService := &corev1.Service{} + if err := kclient.Get(context.TODO(), controller.LoadBalancerServiceName(ic), lbService); err != nil { + t.Fatalf("failed to get LoadBalancer service: %v", err) + } + + ips, err := net.LookupIP(lbService.Status.LoadBalancer.Ingress[0].Hostname) + if err != nil { + t.Logf("[loadbalancer_hostname] failed to resolve %s", lbService.Status.LoadBalancer.Ingress[0].Hostname) + } else { + t.Logf("[loadbalancer_hostname] resolved %s to %v", lbService.Status.LoadBalancer.Ingress[0].Hostname, ips) + } + + testHostname := fmt.Sprintf("test.%s", ic.Spec.Domain) + ips, err = net.LookupIP(testHostname) + if err != nil { + t.Logf("[test_hostname] failed to resolve %s", testHostname) + } else { + t.Logf("[test_hostname] resolved %s to %v", testHostname, ips) + } + + ips, err = net.LookupIP(route.Spec.Host) + if err != nil { + t.Log(err) + return false, nil + } + t.Logf("resolved %s to %v", route.Spec.Host, ips) + + return false, nil + }) + + t.Fatalf("SUCCESS ACTUALLY IM JUST EXITING EARLY SO YOU CAN SEE THE LOGS") } func TestUniqueIdHeader(t *testing.T) { @@ -3850,3 +4139,34 @@ u3YLAbyW/lHhOCiZu2iAI8AbmXem9lW6Tr7p/97s0w== return secret, cl.Create(context.TODO(), secret) } + +func buildExecPodNodeDNS(name, namespace, image string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "execpod", + Image: image, + Command: []string{"/bin/sleep"}, + Args: []string{"4h"}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: pointer.Bool(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + RunAsNonRoot: pointer.Bool(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + DNSPolicy: corev1.DNSDefault, + }, + } +}