From 790dafba74a39fdd9a5cd46aa88593318ba5b02a Mon Sep 17 00:00:00 2001 From: Qi Ni Date: Thu, 13 May 2021 20:47:42 +0800 Subject: [PATCH] chore: add e2e test for byo public IP --- site/content/en/topics/tagging-resources.md | 2 + tests/e2e/network/ensureloadbalancer.go | 107 ++++++++++++++++---- tests/e2e/network/network_security_group.go | 15 ++- tests/e2e/network/service_annotations.go | 88 ++-------------- tests/e2e/utils/network_utils.go | 12 --- tests/e2e/utils/service_utils.go | 107 ++++++++++++++------ 6 files changed, 180 insertions(+), 151 deletions(-) diff --git a/site/content/en/topics/tagging-resources.md b/site/content/en/topics/tagging-resources.md index c6bf0373b5..b21a90ed0b 100644 --- a/site/content/en/topics/tagging-resources.md +++ b/site/content/en/topics/tagging-resources.md @@ -21,6 +21,8 @@ the controller manager would parse this configuration and tag the shared resourc The non-shared resource (public IP) could be tagged by setting `tags` in `azure.json` or service annotation `service.beta.kubernetes.io/azure-pip-tags`. The format of the two is similiar and the tags in the annotation would be considered first when there are conflicts between the configuration file and the annotation. +> The annotation `service.beta.kubernetes.io/azure-pip-tags` only works for managed public IPs. For BYO public IPs, the cloud provider would not apply any tags to them. + When the configuration, file or annotation, is updated, the old ones would be updated if there are conflicts. For example, after updating `{"tags": "a=b,c=d"}` to `{"tags": "a=c,e=f"}`, the new tags would be `a=c,c=d,e=f`. ## Integrating with system tags diff --git a/tests/e2e/network/ensureloadbalancer.go b/tests/e2e/network/ensureloadbalancer.go index 88ae139049..fe2e1bd0b8 100644 --- a/tests/e2e/network/ensureloadbalancer.go +++ b/tests/e2e/network/ensureloadbalancer.go @@ -18,6 +18,7 @@ package network import ( "context" + "fmt" "os" "strings" "time" @@ -87,6 +88,45 @@ var _ = Describe("Ensure LoadBalancer", func() { tc = nil }) + It("should support BYO public IP", func() { + By("creating a public IP with tags") + ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4] + pip := defaultPublicIPAddress(ipName) + expectedTags := map[string]*string{ + "foo": to.StringPtr("bar"), + } + pip.Tags = expectedTags + pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip) + Expect(err).NotTo(HaveOccurred()) + Expect(pip.Tags).To(Equal(expectedTags)) + targetIP := to.String(pip.IPAddress) + utils.Logf("created pip with address %s", targetIP) + + By("creating a service referencing the public IP") + service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, ports) + service = updateServiceBalanceIP(service, false, targetIP) + _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") + Expect(err).NotTo(HaveOccurred()) + Expect(ip).To(Equal(targetIP)) + + By("deleting the service") + err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("test if the pip still exists") + pip, err = utils.WaitGetPIP(tc, ipName) + Expect(err).NotTo(HaveOccurred()) + + By("test if the tags are changed") + Expect(pip.Tags).To(Equal(expectedTags)) + + By("cleaning up") + err = utils.DeletePIPWithRetry(tc, ipName, "") + Expect(err).NotTo(HaveOccurred()) + }) + // Public w/o IP -> Public w/ IP It("should support assigning to specific IP when updating public service", func() { annotation := map[string]string{ @@ -113,7 +153,7 @@ var _ = Describe("Ensure LoadBalancer", func() { }() By("Waiting for exposure of the original service without assigned lb IP") - ip1, err := utils.WaitServiceExposure(cs, ns.Name, testServiceName) + ip1, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") Expect(err).NotTo(HaveOccurred()) Expect(ip1).NotTo(Equal(targetIP)) @@ -126,8 +166,9 @@ var _ = Describe("Ensure LoadBalancer", func() { _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, targetIP, true) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP) Expect(err).NotTo(HaveOccurred()) + Expect(ip).To(Equal(targetIP)) }) // Internal w/ IP -> Internal w/ IP @@ -150,8 +191,9 @@ var _ = Describe("Ensure LoadBalancer", func() { Expect(err).NotTo(HaveOccurred()) }() By("Waiting for exposure of internal service with specific IP") - err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, ip1, true) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, ip1) Expect(err).NotTo(HaveOccurred()) + Expect(ip).To(Equal(ip1)) list, errList := cs.CoreV1().Events(ns.Name).List(context.TODO(), metav1.ListOptions{}) Expect(errList).NotTo(HaveOccurred()) utils.Logf("Events list:") @@ -169,8 +211,9 @@ var _ = Describe("Ensure LoadBalancer", func() { _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, ip2, true) + ip, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, ip2) Expect(err).NotTo(HaveOccurred()) + Expect(ip).To(Equal(ip2)) }) // internal w/o IP -> public w/ IP @@ -198,7 +241,7 @@ var _ = Describe("Ensure LoadBalancer", func() { }() By("Waiting for exposure of the original service without assigned lb private IP") - ip1, err := utils.WaitServiceExposure(cs, ns.Name, testServiceName) + ip1, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") Expect(err).NotTo(HaveOccurred()) Expect(ip1).NotTo(Equal(targetIP)) list, errList := cs.CoreV1().Events(ns.Name).List(context.TODO(), metav1.ListOptions{}) @@ -216,8 +259,9 @@ var _ = Describe("Ensure LoadBalancer", func() { _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, targetIP, true) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP) Expect(err).NotTo(HaveOccurred()) + Expect(ip).To(Equal(targetIP)) }) It("should have no operation since no change in service when update [Slow]", func() { @@ -245,7 +289,7 @@ var _ = Describe("Ensure LoadBalancer", func() { }() By("Waiting for exposure of the original service with assigned lb private IP") - targetIP, err = utils.WaitServiceExposure(cs, ns.Name, testServiceName) + targetIP, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP) Expect(err).NotTo(HaveOccurred()) By("Update without changing the service and wait for a while") @@ -256,8 +300,29 @@ var _ = Describe("Ensure LoadBalancer", func() { _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - //Wait for 10 minutes, there should return timeout err, since external ip should not change - err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, targetIP, false /*expectSame*/) + // Wait for 5 minutes, there should return timeout err, since external ip should not change + err = wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) { + service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) + if err != nil { + if utils.IsRetryableAPIError(err) { + return false, nil + } + return false, err + } + + IngressList := service.Status.LoadBalancer.Ingress + if len(IngressList) == 0 { + err = fmt.Errorf("Cannot find Ingress in limited time") + utils.Logf("Fail to get ingress, retry it in %s seconds", 10) + return false, nil + } + if targetIP == service.Status.LoadBalancer.Ingress[0].IP { + utils.Logf("External IP is still %s", targetIP) + return false, nil + } + utils.Logf("succeeded") + return true, nil + }) Expect(err).To(Equal(wait.ErrWaitTimeout)) }) @@ -271,8 +336,8 @@ var _ = Describe("Ensure LoadBalancer", func() { service1 = updateServiceBalanceIP(service1, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposure(cs, ns.Name, "service1") - Expect(ip).To(Equal(targetIP)) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", targetIP) + Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service1 in namespace %s with IP %s", ns.Name, ip) ports2 := []v1.ServicePort{{ @@ -283,8 +348,8 @@ var _ = Describe("Ensure LoadBalancer", func() { service2 = updateServiceBalanceIP(service2, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err = utils.WaitServiceExposure(cs, ns.Name, "service2") - Expect(ip).To(Equal(targetIP)) + ip, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", targetIP) + Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip) defer func() { @@ -302,7 +367,7 @@ var _ = Describe("Ensure LoadBalancer", func() { service1 := utils.CreateLoadBalancerServiceManifest("service1", nil, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposure(cs, ns.Name, "service1") + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", "") Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service1 in namespace %s with IP %s", ns.Name, ip) @@ -314,17 +379,15 @@ var _ = Describe("Ensure LoadBalancer", func() { service2 = updateServiceBalanceIP(service2, false, ip) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip2, err := utils.WaitServiceExposure(cs, ns.Name, "service2") + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip) Expect(err).NotTo(HaveOccurred()) - Expect(ip2).To(Equal(ip)) utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip) By("Deleting one service and check if the other service works well") err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service1", metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) - ip3, err := utils.WaitServiceExposure(cs, ns.Name, "service2") + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip) Expect(err).NotTo(HaveOccurred()) - Expect(ip3).To(Equal(ip)) By("Deleting all services") err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service2", metav1.DeleteOptions{}) @@ -356,7 +419,7 @@ var _ = Describe("Ensure LoadBalancer", func() { service1 := utils.CreateLoadBalancerServiceManifest("service1", annotation, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposure(cs, ns.Name, "service1") + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", "") Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service1 in namespace %s with IP %s", ns.Name, ip) @@ -368,8 +431,8 @@ var _ = Describe("Ensure LoadBalancer", func() { service2.Spec.LoadBalancerIP = ip _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip2, err := utils.WaitServiceExposure(cs, ns.Name, "service2") - Expect(ip2).To(Equal(ip)) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip) + Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip) defer func() { @@ -394,7 +457,7 @@ var _ = Describe("Ensure LoadBalancer", func() { service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, ports) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - publicIP, err := utils.WaitServiceExposure(cs, ns.Name, testServiceName) + publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") Expect(err).NotTo(HaveOccurred()) By("Checking the initial node number in the LB backend pool") diff --git a/tests/e2e/network/network_security_group.go b/tests/e2e/network/network_security_group.go index e3d759741a..5d5e1a44dd 100644 --- a/tests/e2e/network/network_security_group.go +++ b/tests/e2e/network/network_security_group.go @@ -25,6 +25,7 @@ import ( aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" "github.com/Azure/go-autorest/autorest/to" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -141,7 +142,15 @@ var _ = Describe("Network security group", func() { annotation := map[string]string{ consts.ServiceAnnotationAllowedServiceTag: "AzureCloud", } - _ = createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + utils.Logf("Creating service " + serviceName + " in namespace " + ns.Name) + service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name) + + By("Waiting for the service to be exposed") + _, err = utils.WaitServiceExposure(cs, ns.Name, serviceName, "") + Expect(err).NotTo(HaveOccurred()) By("Validating if the corresponding IP prefix existing in nsg") nsgs, err := tc.GetClusterSecurityGroups() @@ -177,7 +186,7 @@ var _ = Describe("Network security group", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for the service to expose") - internalIP, err := utils.WaitServiceExposure(cs, ns.Name, serviceName) + internalIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") Expect(err).NotTo(HaveOccurred()) By("Checking if there is a deny_all rule") @@ -196,7 +205,7 @@ var _ = Describe("Network security group", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for the service to expose") - internalIP, err = utils.WaitServiceExposure(cs, ns.Name, serviceName) + internalIP, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") Expect(err).NotTo(HaveOccurred()) By("Checking if there is a LoadBalancerSourceRanges rule") diff --git a/tests/e2e/network/service_annotations.go b/tests/e2e/network/service_annotations.go index 6089ba6270..9e0c630b1d 100644 --- a/tests/e2e/network/service_annotations.go +++ b/tests/e2e/network/service_annotations.go @@ -53,8 +53,6 @@ var ( const ( nginxPort = 80 nginxStatusCode = 200 - pullInterval = 20 * time.Second - pullTimeout = 10 * time.Minute testingPort = 81 ) @@ -156,17 +154,12 @@ var _ = Describe("Service with annotation", func() { } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + _ = createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) }() - - By("Validating whether the load balancer is internal") - url := fmt.Sprintf("%s:%v", ip, ports[0].Port) - err := validateInternalLoadBalancer(cs, ns.Name, url) - Expect(err).NotTo(HaveOccurred()) }) It("should support service annotation 'service.beta.kubernetes.io/azure-load-balancer-internal-subnet'", func() { @@ -287,7 +280,7 @@ var _ = Describe("Service with annotation", func() { //wait and get service's public IP Address By("Waiting service to expose...") - _, err = utils.WaitServiceExposure(cs, ns.Name, serviceName) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, to.String(pip.IPAddress)) Expect(err).NotTo(HaveOccurred()) lb := getAzureLoadBalancerFromPIP(tc, *pip.IPAddress, *rg.Name, "") @@ -333,7 +326,7 @@ var _ = Describe("Service with annotation", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting service to expose...") - ip, err := utils.WaitServiceExposure(cs, ns.Name, serviceName) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") Expect(err).NotTo(HaveOccurred()) defer func() { @@ -400,7 +393,7 @@ var _ = Describe("Service with annotation", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for the service to expose") - ip, err := utils.WaitServiceExposure(cs, ns.Name, serviceName) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") Expect(err).NotTo(HaveOccurred()) Expect(ip).To(Equal(to.String(pip1.IPAddress))) @@ -412,7 +405,7 @@ var _ = Describe("Service with annotation", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for service IP to be updated") - err = utils.WaitServiceIPEqualTo(cs, to.String(pip2.IPAddress), serviceName, ns.Name) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, to.String(pip2.IPAddress)) Expect(err).NotTo(HaveOccurred()) }) }) @@ -564,7 +557,7 @@ func createAndExposeDefaultServiceWithAnnotation(cs clientset.Interface, service //wait and get service's public IP Address utils.Logf("Waiting service to expose...") - publicIP, err := utils.WaitServiceExposure(cs, nsName, serviceName) + publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, nsName, serviceName, "") Expect(err).NotTo(HaveOccurred()) return publicIP @@ -607,73 +600,6 @@ func createNginxDeploymentManifest(name string, labels map[string]string) *appsv } } -// validate internal source can access to ILB -// nolint:unused -func validateInternalLoadBalancer(c clientset.Interface, ns string, url string) error { - // create a pod to access to the service - utils.Logf("Validating external IP not be public and internal accessible") - utils.Logf("Create a front pod to connect to service") - podName := "front-pod" - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - }, - Spec: v1.PodSpec{ - Hostname: podName, - Containers: []v1.Container{ - { - Name: "test-app", - Image: "appropriate/curl", - ImagePullPolicy: v1.PullIfNotPresent, - Command: []string{ - "/bin/sh", - "-c", - "code=0; while [ $code != 200 ]; do code=$(curl -s -o /dev/null -w \"%{http_code}\" " + url + "); sleep 1; done; echo $code", - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, - }, - } - _, err := c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) - if err != nil { - return err - } - defer func() { - utils.Logf("Deleting front pod") - err = utils.DeletePod(c, ns, podName) - }() - - // publicFlag shows whether public accessible test ends - // internalFlag shows whether internal accessible test ends - utils.Logf("Call from the created pod") - err = wait.PollImmediate(pullInterval, pullTimeout, func() (bool, error) { - pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{}) - if err != nil { - if utils.IsRetryableAPIError(err) { - return false, nil - } - return false, err - } - if pod.Status.Phase != v1.PodSucceeded { - utils.Logf("waiting for the pod succeeded") - return false, nil - } - if pod.Status.ContainerStatuses[0].State.Terminated == nil || pod.Status.ContainerStatuses[0].State.Terminated.Reason != "Completed" { - utils.Logf("waiting for the container completed") - return false, nil - } - utils.Logf("Still testing internal access from front pod to internal service") - log, err := c.CoreV1().Pods(ns).GetLogs(pod.Name, &v1.PodLogOptions{}).Do(context.TODO()).Raw() - if err != nil { - return false, nil - } - return strings.Contains(fmt.Sprintf("%s", log), "200"), nil - }) - utils.Logf("validation finished") - return err -} - func validateLoadBalancerBackendPools(tc *utils.AzureTestClient, vmssName string, cs clientset.Interface, serviceName string, labels map[string]string, ns string, ports []v1.ServicePort, resourceGroupName string) { serviceName = fmt.Sprintf("%s-%s", serviceName, vmssName) @@ -689,7 +615,7 @@ func validateLoadBalancerBackendPools(tc *utils.AzureTestClient, vmssName string //wait and get service's public IP Address By("Waiting for service exposure") - publicIP, err := utils.WaitServiceExposure(cs, ns, serviceName) + publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns, serviceName, "") Expect(err).NotTo(HaveOccurred()) // Invoking azure network client to get list of public IP Addresses diff --git a/tests/e2e/utils/network_utils.go b/tests/e2e/utils/network_utils.go index d4f4824772..9f540cb8c8 100644 --- a/tests/e2e/utils/network_utils.go +++ b/tests/e2e/utils/network_utils.go @@ -21,7 +21,6 @@ import ( "fmt" "net" "strings" - "time" aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" "github.com/Azure/go-autorest/autorest/to" @@ -29,7 +28,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" ) // getVirtualNetworkList returns the list of virtual networks in the cluster resource group. @@ -294,13 +292,3 @@ func (azureTestClient *AzureTestClient) GetLoadBalancer(resourceGroupName, lbNam lbClient := azureTestClient.creteLoadBalancerClient() return lbClient.Get(context.Background(), resourceGroupName, lbName, "") } - -func WaitServiceIPEqualTo(cs clientset.Interface, expectedIP, serviceName, namespace string) error { - return wait.PollImmediate(10*time.Second, 10*time.Minute, func() (done bool, err error) { - ip, err := WaitServiceExposure(cs, namespace, serviceName) - if err != nil { - return false, err - } - return strings.EqualFold(ip, expectedIP), nil - }) -} diff --git a/tests/e2e/utils/service_utils.go b/tests/e2e/utils/service_utils.go index 01d96191c9..d79c94f0ad 100644 --- a/tests/e2e/utils/service_utils.go +++ b/tests/e2e/utils/service_utils.go @@ -19,10 +19,13 @@ package utils import ( "context" "fmt" + "net/http" "os" "strings" "time" + "sigs.k8s.io/cloud-provider-azure/pkg/consts" + aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" v1 "k8s.io/api/core/v1" @@ -35,6 +38,8 @@ import ( const ( serviceTimeout = 5 * time.Minute serviceTimeoutBasicLB = 10 * time.Minute + pullInterval = 20 * time.Second + pullTimeout = 1 * time.Minute ) // DeleteService deletes a service @@ -71,10 +76,35 @@ func GetServiceDomainName(prefix string) (ret string) { return } -// WaitServiceExposure returns ip of ingress -func WaitServiceExposure(cs clientset.Interface, namespace string, name string) (string, error) { +// WaitServiceExposureAndValidateConnectivity returns ip of the service and check the connectivity if it is a public IP +func WaitServiceExposureAndValidateConnectivity(cs clientset.Interface, namespace string, name string, targetIP string) (string, error) { + var service *v1.Service + var err error + var ip string + + service, err = WaitServiceExposure(cs, namespace, name, targetIP) + if err != nil { + return "", err + } + + ip = service.Status.LoadBalancer.Ingress[0].IP + + if !isInternalService(service) { + Logf("checking the connectivity of the public IP %s", ip) + port := service.Spec.Ports[0].Port + if err := ValidateExternalServiceConnectivity(ip, int(port)); err != nil { + return ip, err + } + } + + return ip, nil +} + +// WaitServiceExposure waits for the exposure of the external IP of the service +func WaitServiceExposure(cs clientset.Interface, namespace string, name string, targetIP string) (*v1.Service, error) { var service *v1.Service var err error + var ip string timeout := serviceTimeout if skuEnv := os.Getenv(LoadBalancerSkuEnv); skuEnv != "" { @@ -95,51 +125,62 @@ func WaitServiceExposure(cs clientset.Interface, namespace string, name string) IngressList := service.Status.LoadBalancer.Ingress if len(IngressList) == 0 { err = fmt.Errorf("Cannot find Ingress in limited time") - Logf("Fail to find ingress, retry it in 10 seconds") + Logf("Fail to find ingress, retry in 10 seconds") + return false, nil + } + + ip = service.Status.LoadBalancer.Ingress[0].IP + if targetIP != "" && !strings.EqualFold(ip, targetIP) { + Logf("expected IP is %s, current IP is %s, retry in 10 seconds", targetIP, ip) return false, nil } + return true, nil }) != nil { - return "", err + return nil, err } - ip := service.Status.LoadBalancer.Ingress[0].IP + Logf("Exposure successfully, get external ip: %s", ip) - return ip, nil + return service, nil } -// WaitUpdateServiceExposure returns ip of ingress -func WaitUpdateServiceExposure(cs clientset.Interface, namespace string, name string, targetIP string, expectSame bool) error { - var service *v1.Service - var err error - poll := 10 * time.Second - timeout := 10 * time.Minute +func isInternalService(service *v1.Service) bool { + var ( + val string + ok bool + ) + if val, ok = service.Annotations[consts.ServiceAnnotationLoadBalancerInternal]; !ok { + return false + } - return wait.PollImmediate(poll, timeout, func() (bool, error) { - service, err = cs.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - if IsRetryableAPIError(err) { - return false, nil - } - return false, err - } + return strings.EqualFold(val, "true") +} - IngressList := service.Status.LoadBalancer.Ingress - if len(IngressList) == 0 { - err = fmt.Errorf("Cannot find Ingress in limited time") - Logf("Fail to get ingress, retry it in %v seconds", poll) +// ValidateExternalServiceConnectivity validates the connectivity of the service IP +func ValidateExternalServiceConnectivity(serviceIP string, port int) error { + // the default nginx port is 80, skip other ports + if port != 80 { + return nil + } + + err := wait.PollImmediate(pullInterval, pullTimeout, func() (done bool, err error) { + resp, err := http.Get(fmt.Sprintf("http://%s:%d", serviceIP, port)) + if err != nil { + Logf("got error %v, will retry", err) return false, nil } - if targetIP != service.Status.LoadBalancer.Ingress[0].IP == expectSame { - if expectSame { - Logf("still unmatched external IP, retry it in %v seconds", poll) - } else { - Logf("External IP is still %s", targetIP) - } - return false, nil + + if 200 <= resp.StatusCode && resp.StatusCode < 300 { + Logf("succeeded") + return true, nil } - Logf("Exposure successfully") - return true, nil + + Logf("got status code %d", resp.StatusCode) + return false, nil }) + + Logf("validation finished") + return err } // extractSuffix obtains the server domain name suffix