diff --git a/site/content/en/topics/tagging-resources.md b/site/content/en/topics/tagging-resources.md index 118fc1cac0..1b3b6a43d4 100644 --- a/site/content/en/topics/tagging-resources.md +++ b/site/content/en/topics/tagging-resources.md @@ -21,6 +21,8 @@ the controller manager would parse this configuration and tag the shared resourc The non-shared resource (public IP) could be tagged by setting `tags` in `azure.json` or service annotation `service.beta.kubernetes.io/azure-pip-tags`. The format of the two is similar and the tags in the annotation would be considered first when there are conflicts between the configuration file and the annotation. +> The annotation `service.beta.kubernetes.io/azure-pip-tags` only works for managed public IPs. For BYO public IPs, the cloud provider would not apply any tags to them. + When the configuration, file or annotation, is updated, the old ones would be updated if there are conflicts. For example, after updating `{"tags": "a=b,c=d"}` to `{"tags": "a=c,e=f"}`, the new tags would be `a=c,c=d,e=f`. ## Integrating with system tags diff --git a/tests/e2e/network/ensureloadbalancer.go b/tests/e2e/network/ensureloadbalancer.go index 5494a4816c..9f9b84fae5 100644 --- a/tests/e2e/network/ensureloadbalancer.go +++ b/tests/e2e/network/ensureloadbalancer.go @@ -18,6 +18,7 @@ package network import ( "context" + "fmt" "os" "strings" "time" @@ -96,6 +97,45 @@ var _ = Describe("Ensure LoadBalancer", func() { tc = nil }) + It("should support BYO public IP", func() { + By("creating a public IP with tags") + ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4] + pip := defaultPublicIPAddress(ipName) + expectedTags := map[string]*string{ + "foo": to.StringPtr("bar"), + } + pip.Tags = expectedTags + pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip) + Expect(err).NotTo(HaveOccurred()) + Expect(pip.Tags).To(Equal(expectedTags)) + targetIP := to.String(pip.IPAddress) + utils.Logf("created pip with address %s", targetIP) + + By("creating a service referencing the public IP") + service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, ports) + service = updateServiceBalanceIP(service, false, targetIP) + _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") + Expect(err).NotTo(HaveOccurred()) + Expect(ip).To(Equal(targetIP)) + + By("deleting the service") + err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("test if the pip still exists") + pip, err = utils.WaitGetPIP(tc, ipName) + Expect(err).NotTo(HaveOccurred()) + + By("test if the tags are changed") + Expect(pip.Tags).To(Equal(expectedTags)) + + By("cleaning up") + err = utils.DeletePIPWithRetry(tc, ipName, "") + Expect(err).NotTo(HaveOccurred()) + }) + // Public w/o IP -> Public w/ IP It("should support assigning to specific IP when updating public service", func() { ipName := basename + "-public-none-IP" + string(uuid.NewUUID())[0:4] @@ -119,7 +159,7 @@ var _ = Describe("Ensure LoadBalancer", func() { }() By("Waiting for exposure of the original service without assigned lb IP") - ip1, err := utils.WaitServiceExposure(cs, ns.Name, testServiceName) + ip1, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") Expect(err).NotTo(HaveOccurred()) Expect(ip1).NotTo(Equal(targetIP)) @@ -132,8 +172,9 @@ var _ = Describe("Ensure LoadBalancer", func() { _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, targetIP, true) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP) Expect(err).NotTo(HaveOccurred()) + Expect(ip).To(Equal(targetIP)) }) // Internal w/ IP -> Internal w/ IP @@ -153,8 +194,9 @@ var _ = Describe("Ensure LoadBalancer", func() { Expect(err).NotTo(HaveOccurred()) }() By("Waiting for exposure of internal service with specific IP") - err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, ip1, true) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, ip1) Expect(err).NotTo(HaveOccurred()) + Expect(ip).To(Equal(ip1)) list, errList := cs.CoreV1().Events(ns.Name).List(context.TODO(), metav1.ListOptions{}) Expect(errList).NotTo(HaveOccurred()) utils.Logf("Events list:") @@ -172,8 +214,9 @@ var _ = Describe("Ensure LoadBalancer", func() { _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, ip2, true) + ip, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, ip2) Expect(err).NotTo(HaveOccurred()) + Expect(ip).To(Equal(ip2)) }) // internal w/o IP -> public w/ IP @@ -198,7 +241,7 @@ var _ = Describe("Ensure LoadBalancer", func() { }() By("Waiting for exposure of the original service without assigned lb private IP") - ip1, err := utils.WaitServiceExposure(cs, ns.Name, testServiceName) + ip1, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") Expect(err).NotTo(HaveOccurred()) Expect(ip1).NotTo(Equal(targetIP)) list, errList := cs.CoreV1().Events(ns.Name).List(context.TODO(), metav1.ListOptions{}) @@ -216,8 +259,9 @@ var _ = Describe("Ensure LoadBalancer", func() { _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, targetIP, true) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP) Expect(err).NotTo(HaveOccurred()) + Expect(ip).To(Equal(targetIP)) }) It("should have no operation since no change in service when update [Slow]", func() { @@ -242,7 +286,7 @@ var _ = Describe("Ensure LoadBalancer", func() { }() By("Waiting for exposure of the original service with assigned lb private IP") - targetIP, err = utils.WaitServiceExposure(cs, ns.Name, testServiceName) + targetIP, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP) Expect(err).NotTo(HaveOccurred()) By("Update without changing the service and wait for a while") @@ -253,8 +297,29 @@ var _ = Describe("Ensure LoadBalancer", func() { _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - //Wait for 10 minutes, there should return timeout err, since external ip should not change - err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, targetIP, false /*expectSame*/) + // Wait for 5 minutes, there should return timeout err, since external ip should not change + err = wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) { + service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) + if err != nil { + if utils.IsRetryableAPIError(err) { + return false, nil + } + return false, err + } + + IngressList := service.Status.LoadBalancer.Ingress + if len(IngressList) == 0 { + err = fmt.Errorf("Cannot find Ingress in limited time") + utils.Logf("Fail to get ingress, retry it in %s seconds", 10) + return false, nil + } + if targetIP == service.Status.LoadBalancer.Ingress[0].IP { + utils.Logf("External IP is still %s", targetIP) + return false, nil + } + utils.Logf("succeeded") + return true, nil + }) Expect(err).To(Equal(wait.ErrWaitTimeout)) }) @@ -268,10 +333,8 @@ var _ = Describe("Ensure LoadBalancer", func() { service1 = updateServiceBalanceIP(service1, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - var ip string - ip, err = utils.WaitServiceExposure(cs, ns.Name, "service1") + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", targetIP) Expect(err).NotTo(HaveOccurred()) - Expect(ip).To(Equal(targetIP)) utils.Logf("Successfully created LoadBalancer service1 in namespace %s with IP %s", ns.Name, ip) ports2 := []v1.ServicePort{{ @@ -282,8 +345,8 @@ var _ = Describe("Ensure LoadBalancer", func() { service2 = updateServiceBalanceIP(service2, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err = utils.WaitServiceExposure(cs, ns.Name, "service2") - Expect(ip).To(Equal(targetIP)) + ip, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", targetIP) + Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip) defer func() { @@ -301,7 +364,7 @@ var _ = Describe("Ensure LoadBalancer", func() { service1 := utils.CreateLoadBalancerServiceManifest("service1", nil, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposure(cs, ns.Name, "service1") + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", "") Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service1 in namespace %s with IP %s", ns.Name, ip) @@ -313,17 +376,15 @@ var _ = Describe("Ensure LoadBalancer", func() { service2 = updateServiceBalanceIP(service2, false, ip) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip2, err := utils.WaitServiceExposure(cs, ns.Name, "service2") + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip) Expect(err).NotTo(HaveOccurred()) - Expect(ip2).To(Equal(ip)) utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip) By("Deleting one service and check if the other service works well") err = utils.DeleteService(cs, ns.Name, "service1") Expect(err).NotTo(HaveOccurred()) - ip3, err := utils.WaitServiceExposure(cs, ns.Name, "service2") + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip) Expect(err).NotTo(HaveOccurred()) - Expect(ip3).To(Equal(ip)) By("Deleting all services") err = utils.DeleteService(cs, ns.Name, "service2") @@ -352,7 +413,7 @@ var _ = Describe("Ensure LoadBalancer", func() { service1 := utils.CreateLoadBalancerServiceManifest("service1", serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposure(cs, ns.Name, "service1") + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", "") Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service1 in namespace %s with IP %s", ns.Name, ip) @@ -364,8 +425,8 @@ var _ = Describe("Ensure LoadBalancer", func() { service2.Spec.LoadBalancerIP = ip _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip2, err := utils.WaitServiceExposure(cs, ns.Name, "service2") - Expect(ip2).To(Equal(ip)) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip) + Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip) defer func() { @@ -391,7 +452,7 @@ var _ = Describe("Ensure LoadBalancer", func() { service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, ports) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - publicIP, err := utils.WaitServiceExposure(cs, ns.Name, testServiceName) + publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") Expect(err).NotTo(HaveOccurred()) By("Checking the initial node number in the LB backend pool") diff --git a/tests/e2e/network/network_security_group.go b/tests/e2e/network/network_security_group.go index e3d759741a..5d5e1a44dd 100644 --- a/tests/e2e/network/network_security_group.go +++ b/tests/e2e/network/network_security_group.go @@ -25,6 +25,7 @@ import ( aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" "github.com/Azure/go-autorest/autorest/to" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -141,7 +142,15 @@ var _ = Describe("Network security group", func() { annotation := map[string]string{ consts.ServiceAnnotationAllowedServiceTag: "AzureCloud", } - _ = createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + utils.Logf("Creating service " + serviceName + " in namespace " + ns.Name) + service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name) + + By("Waiting for the service to be exposed") + _, err = utils.WaitServiceExposure(cs, ns.Name, serviceName, "") + Expect(err).NotTo(HaveOccurred()) By("Validating if the corresponding IP prefix existing in nsg") nsgs, err := tc.GetClusterSecurityGroups() @@ -177,7 +186,7 @@ var _ = Describe("Network security group", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for the service to expose") - internalIP, err := utils.WaitServiceExposure(cs, ns.Name, serviceName) + internalIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") Expect(err).NotTo(HaveOccurred()) By("Checking if there is a deny_all rule") @@ -196,7 +205,7 @@ var _ = Describe("Network security group", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for the service to expose") - internalIP, err = utils.WaitServiceExposure(cs, ns.Name, serviceName) + internalIP, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") Expect(err).NotTo(HaveOccurred()) By("Checking if there is a LoadBalancerSourceRanges rule") diff --git a/tests/e2e/network/service_annotations.go b/tests/e2e/network/service_annotations.go index dffd2f6ab1..0237a2e1a7 100644 --- a/tests/e2e/network/service_annotations.go +++ b/tests/e2e/network/service_annotations.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" "sigs.k8s.io/cloud-provider-azure/pkg/consts" "sigs.k8s.io/cloud-provider-azure/tests/e2e/utils" @@ -53,8 +54,6 @@ var ( const ( nginxPort = 80 nginxStatusCode = 200 - pullInterval = 20 * time.Second - pullTimeout = 10 * time.Minute testingPort = 81 ) @@ -160,17 +159,12 @@ var _ = Describe("Service with annotation", func() { } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + _ = createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) }() - - By("Validating whether the load balancer is internal") - url := fmt.Sprintf("%s:%v", ip, ports[0].Port) - err := validateInternalLoadBalancer(cs, ns.Name, url) - Expect(err).NotTo(HaveOccurred()) }) It("should support service annotation 'service.beta.kubernetes.io/azure-load-balancer-internal-subnet'", func() { @@ -291,7 +285,7 @@ var _ = Describe("Service with annotation", func() { //wait and get service's public IP Address By("Waiting service to expose...") - _, err = utils.WaitServiceExposure(cs, ns.Name, serviceName) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, to.String(pip.IPAddress)) Expect(err).NotTo(HaveOccurred()) lb := getAzureLoadBalancerFromPIP(tc, *pip.IPAddress, *rg.Name, "") @@ -337,7 +331,7 @@ var _ = Describe("Service with annotation", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting service to expose...") - ip, err := utils.WaitServiceExposure(cs, ns.Name, serviceName) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") Expect(err).NotTo(HaveOccurred()) defer func() { @@ -406,7 +400,7 @@ var _ = Describe("Service with annotation", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for the service to expose") - ip, err := utils.WaitServiceExposure(cs, ns.Name, serviceName) + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") Expect(err).NotTo(HaveOccurred()) Expect(ip).To(Equal(to.String(pip1.IPAddress))) @@ -418,7 +412,7 @@ var _ = Describe("Service with annotation", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for service IP to be updated") - err = utils.WaitServiceIPEqualTo(cs, to.String(pip2.IPAddress), serviceName, ns.Name) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, to.String(pip2.IPAddress)) Expect(err).NotTo(HaveOccurred()) }) @@ -438,8 +432,14 @@ var _ = Describe("Service with annotation", func() { err = utils.DeletePIPWithRetry(tc, publicIP, tc.GetResourceGroup()) Expect(err).NotTo(HaveOccurred()) }() - // get lb from azure client - lb := getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") + + var lb *network.LoadBalancer + //wait for backend update + err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) { + lb = getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") + return len(*lb.LoadBalancerPropertiesFormat.Probes) == 1, nil + }) + Expect(err).NotTo(HaveOccurred()) By("Validating health probe configs") var numberOfProbes *int32 @@ -466,8 +466,15 @@ var _ = Describe("Service with annotation", func() { err = utils.DeletePIPWithRetry(tc, publicIP, tc.GetResourceGroup()) Expect(err).NotTo(HaveOccurred()) }() + + var lb *network.LoadBalancer + //wait for backend update + err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) { + lb = getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") + return len(*lb.LoadBalancerPropertiesFormat.Probes) == 1, nil + }) + Expect(err).NotTo(HaveOccurred()) // get lb from azure client - lb := getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") By("Validating health probe configs") probes := *lb.Probes Expect((len(probes))).To(Equal(1)) @@ -476,8 +483,8 @@ var _ = Describe("Service with annotation", func() { }) var _ = Describe("[[Multi-Nodepool]][VMSS]", func() { - basename := "service" - serviceName := "annotation-test" + basename := "vmssservice" + serviceName := "vmss-test" var ( cs clientset.Interface @@ -560,6 +567,153 @@ var _ = Describe("[[Multi-Nodepool]][VMSS]", func() { }) }) +var _ = Describe("Multi-ports service", func() { + basename := "mpservice" + serviceName := "multiport-test" + + var ( + cs clientset.Interface + tc *utils.AzureTestClient + ns *v1.Namespace + ) + + labels := map[string]string{ + "app": serviceName, + } + ports := []v1.ServicePort{{ + AppProtocol: to.StringPtr("Tcp"), + Port: nginxPort, + Name: "port1", + TargetPort: intstr.FromInt(nginxPort), + }, { + Port: nginxPort + 1, + Name: "port2", + TargetPort: intstr.FromInt(nginxPort), + AppProtocol: to.StringPtr("Tcp"), + }, + } + + BeforeEach(func() { + var err error + cs, err = utils.CreateKubeClientSet() + Expect(err).NotTo(HaveOccurred()) + + ns, err = utils.CreateTestingNamespace(basename, cs) + Expect(err).NotTo(HaveOccurred()) + + utils.Logf("Creating deployment " + serviceName) + deployment := createNginxDeploymentManifest(serviceName, labels) + _, err = cs.AppsV1().Deployments(ns.Name).Create(context.TODO(), deployment, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + utils.Logf("Waiting for backend pods to be ready") + err = utils.WaitPodsToBeReady(cs, ns.Name) + Expect(err).NotTo(HaveOccurred()) + + utils.Logf("Creating Azure clients") + tc, err = utils.CreateAzureTestClient() + Expect(err).NotTo(HaveOccurred()) + + }) + + AfterEach(func() { + err := cs.AppsV1().Deployments(ns.Name).Delete(context.TODO(), serviceName, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + err = utils.DeleteNamespace(cs, ns.Name) + Expect(err).NotTo(HaveOccurred()) + + cs = nil + ns = nil + tc = nil + }) + Context("When ExternalTrafficPolicy is updated", func() { + It("Should not have error occurred", func() { + By("Getting the service") + annotation := map[string]string{} + utils.Logf("Creating service " + serviceName + " in namespace " + ns.Name) + service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) + service, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name) + + //wait and get service's public IP Address + utils.Logf("Waiting service to expose...") + publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + Expect(err).NotTo(HaveOccurred()) + // create service with given annotation and wait it to expose + + defer func() { + By("Cleaning up service and public IP") + err := utils.DeleteService(cs, ns.Name, serviceName) + Expect(err).NotTo(HaveOccurred()) + err = utils.DeletePIPWithRetry(tc, publicIP, tc.GetResourceGroup()) + Expect(err).NotTo(HaveOccurred()) + }() + + By("Changing ExternalTrafficPolicy of the service to Local") + + utils.Logf("Updating service " + serviceName + " in namespace " + ns.Name) + retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { + service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), serviceName, metav1.GetOptions{}) + if err != nil { + return err + } + service.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal + _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) + return err + }) + Expect(retryErr).NotTo(HaveOccurred()) + utils.Logf("Successfully updated LoadBalancer service " + serviceName + " in namespace " + ns.Name) + + By("Getting updated service object from server") + retryErr = retry.RetryOnConflict(retry.DefaultRetry, func() error { + service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), serviceName, metav1.GetOptions{}) + if err != nil { + return err + } + if service.Spec.HealthCheckNodePort == 0 { + return fmt.Errorf("service HealthCheckNodePort is not updated") + } + return nil + }) + Expect(retryErr).NotTo(HaveOccurred()) + + var lb *network.LoadBalancer + //wait for backend update + err = wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) { + lb = getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") + return len(*lb.LoadBalancerPropertiesFormat.Probes) == 1 && *(*lb.LoadBalancerPropertiesFormat.Probes)[0].Port == service.Spec.HealthCheckNodePort, nil + }) + Expect(err).NotTo(HaveOccurred()) + + var nodeHealthCheckPort = service.Spec.HealthCheckNodePort + By("Changing ExternalTrafficPolicy of the service to Cluster") + utils.Logf("Updating service " + serviceName + " in namespace " + ns.Name) + retryErr = retry.RetryOnConflict(retry.DefaultRetry, func() error { + service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), serviceName, metav1.GetOptions{}) + if err != nil { + return err + } + service.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster + _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) + return err + }) + Expect(retryErr).NotTo(HaveOccurred()) + utils.Logf("Successfully updated LoadBalancer service " + serviceName + " in namespace " + ns.Name) + + //wait for backend update + err = wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) { + lb := getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") + return len(*lb.LoadBalancerPropertiesFormat.Probes) == 2 && + *(*lb.LoadBalancerPropertiesFormat.Probes)[0].Port != nodeHealthCheckPort && + *(*lb.LoadBalancerPropertiesFormat.Probes)[1].Port != nodeHealthCheckPort, nil + }) + Expect(err).NotTo(HaveOccurred()) + }) + }) +}) + func waitComparePIPTags(tc *utils.AzureTestClient, expectedTags map[string]*string, pipName string) error { err := wait.PollImmediate(10*time.Second, 10*time.Minute, func() (done bool, err error) { pip, err := utils.WaitGetPIP(tc, pipName) @@ -622,7 +776,7 @@ func createAndExposeDefaultServiceWithAnnotation(cs clientset.Interface, service //wait and get service's public IP Address utils.Logf("Waiting service to expose...") - publicIP, err := utils.WaitServiceExposure(cs, nsName, serviceName) + publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, nsName, serviceName, "") Expect(err).NotTo(HaveOccurred()) return publicIP @@ -665,73 +819,6 @@ func createNginxDeploymentManifest(name string, labels map[string]string) *appsv } } -// validate internal source can access to ILB -// nolint:unused -func validateInternalLoadBalancer(c clientset.Interface, ns string, url string) error { - // create a pod to access to the service - utils.Logf("Validating external IP not be public and internal accessible") - utils.Logf("Create a front pod to connect to service") - podName := "front-pod" - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - }, - Spec: v1.PodSpec{ - Hostname: podName, - Containers: []v1.Container{ - { - Name: "test-app", - Image: "appropriate/curl", - ImagePullPolicy: v1.PullIfNotPresent, - Command: []string{ - "/bin/sh", - "-c", - "code=0; while [ $code != 200 ]; do code=$(curl -s -o /dev/null -w \"%{http_code}\" " + url + "); sleep 1; done; echo $code", - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, - }, - } - _, err := c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) - if err != nil { - return err - } - defer func() { - utils.Logf("Deleting front pod") - err = utils.DeletePod(c, ns, podName) - }() - - // publicFlag shows whether public accessible test ends - // internalFlag shows whether internal accessible test ends - utils.Logf("Call from the created pod") - err = wait.PollImmediate(pullInterval, pullTimeout, func() (bool, error) { - pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{}) - if err != nil { - if utils.IsRetryableAPIError(err) { - return false, nil - } - return false, err - } - if pod.Status.Phase != v1.PodSucceeded { - utils.Logf("waiting for the pod succeeded") - return false, nil - } - if pod.Status.ContainerStatuses[0].State.Terminated == nil || pod.Status.ContainerStatuses[0].State.Terminated.Reason != "Completed" { - utils.Logf("waiting for the container completed") - return false, nil - } - utils.Logf("Still testing internal access from front pod to internal service") - log, err := c.CoreV1().Pods(ns).GetLogs(pod.Name, &v1.PodLogOptions{}).Do(context.TODO()).Raw() - if err != nil { - return false, nil - } - return strings.Contains(string(log), "200"), nil - }) - utils.Logf("validation finished") - return err -} - func validateLoadBalancerBackendPools(tc *utils.AzureTestClient, vmssName string, cs clientset.Interface, serviceName string, labels map[string]string, ns string, ports []v1.ServicePort, resourceGroupName string) { serviceName = fmt.Sprintf("%s-%s", serviceName, vmssName) @@ -747,7 +834,7 @@ func validateLoadBalancerBackendPools(tc *utils.AzureTestClient, vmssName string //wait and get service's public IP Address By("Waiting for service exposure") - publicIP, err := utils.WaitServiceExposure(cs, ns, serviceName) + publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns, serviceName, "") Expect(err).NotTo(HaveOccurred()) // Invoking azure network client to get list of public IP Addresses diff --git a/tests/e2e/utils/network_utils.go b/tests/e2e/utils/network_utils.go index 1e8c204d9d..e12651ab3d 100644 --- a/tests/e2e/utils/network_utils.go +++ b/tests/e2e/utils/network_utils.go @@ -21,7 +21,6 @@ import ( "fmt" "net" "strings" - "time" aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" "github.com/Azure/go-autorest/autorest/to" @@ -29,7 +28,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" ) // getVirtualNetworkList returns the list of virtual networks in the cluster resource group. @@ -294,13 +292,3 @@ func (azureTestClient *AzureTestClient) GetLoadBalancer(resourceGroupName, lbNam lbClient := azureTestClient.creteLoadBalancerClient() return lbClient.Get(context.Background(), resourceGroupName, lbName, "") } - -func WaitServiceIPEqualTo(cs clientset.Interface, expectedIP, serviceName, namespace string) error { - return wait.PollImmediate(10*time.Second, 10*time.Minute, func() (done bool, err error) { - ip, err := WaitServiceExposure(cs, namespace, serviceName) - if err != nil { - return false, err - } - return strings.EqualFold(ip, expectedIP), nil - }) -} diff --git a/tests/e2e/utils/service_utils.go b/tests/e2e/utils/service_utils.go index 0ecc4a818b..13ad8c3874 100644 --- a/tests/e2e/utils/service_utils.go +++ b/tests/e2e/utils/service_utils.go @@ -19,10 +19,13 @@ package utils import ( "context" "fmt" + "net/http" "os" "strings" "time" + "sigs.k8s.io/cloud-provider-azure/pkg/consts" + aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" v1 "k8s.io/api/core/v1" @@ -35,6 +38,8 @@ import ( const ( serviceTimeout = 5 * time.Minute serviceTimeoutBasicLB = 10 * time.Minute + pullInterval = 20 * time.Second + pullTimeout = 1 * time.Minute ) // DeleteService deletes a service @@ -70,10 +75,36 @@ func GetServiceDomainName(prefix string) (ret string) { return } -// WaitServiceExposure returns ip of ingress -func WaitServiceExposure(cs clientset.Interface, namespace string, name string) (string, error) { +// WaitServiceExposureAndValidateConnectivity returns ip of the service and check the connectivity if it is a public IP +func WaitServiceExposureAndValidateConnectivity(cs clientset.Interface, namespace string, name string, targetIP string) (string, error) { var service *v1.Service var err error + var ip string + + service, err = WaitServiceExposure(cs, namespace, name, targetIP) + if err != nil { + return "", err + } + + ip = service.Status.LoadBalancer.Ingress[0].IP + + if !isInternalService(service) { + Logf("checking the connectivity of the public IP %s", ip) + for _, port := range service.Spec.Ports { + if err := ValidateExternalServiceConnectivity(ip, int(port.Port)); err != nil { + return ip, err + } + } + } + + return ip, nil +} + +// WaitServiceExposure waits for the exposure of the external IP of the service +func WaitServiceExposure(cs clientset.Interface, namespace string, name string, targetIP string) (*v1.Service, error) { + var service *v1.Service + var err error + var ip string timeout := serviceTimeout if skuEnv := os.Getenv(LoadBalancerSkuEnv); skuEnv != "" { @@ -94,51 +125,62 @@ func WaitServiceExposure(cs clientset.Interface, namespace string, name string) IngressList := service.Status.LoadBalancer.Ingress if len(IngressList) == 0 { err = fmt.Errorf("Cannot find Ingress in limited time") - Logf("Fail to find ingress, retry it in 10 seconds") + Logf("Fail to find ingress, retry in 10 seconds") + return false, nil + } + + ip = service.Status.LoadBalancer.Ingress[0].IP + if targetIP != "" && !strings.EqualFold(ip, targetIP) { + Logf("expected IP is %s, current IP is %s, retry in 10 seconds", targetIP, ip) return false, nil } + return true, nil }) != nil { - return "", err + return nil, err } - ip := service.Status.LoadBalancer.Ingress[0].IP + Logf("Exposure successfully, get external ip: %s", ip) - return ip, nil + return service, nil } -// WaitUpdateServiceExposure returns ip of ingress -func WaitUpdateServiceExposure(cs clientset.Interface, namespace string, name string, targetIP string, expectSame bool) error { - var service *v1.Service - var err error - poll := 10 * time.Second - timeout := 10 * time.Minute +func isInternalService(service *v1.Service) bool { + var ( + val string + ok bool + ) + if val, ok = service.Annotations[consts.ServiceAnnotationLoadBalancerInternal]; !ok { + return false + } - return wait.PollImmediate(poll, timeout, func() (bool, error) { - service, err = cs.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - if IsRetryableAPIError(err) { - return false, nil - } - return false, err - } + return strings.EqualFold(val, "true") +} - IngressList := service.Status.LoadBalancer.Ingress - if len(IngressList) == 0 { - err = fmt.Errorf("Cannot find Ingress in limited time") - Logf("Fail to get ingress, retry it in %v seconds", poll) +// ValidateExternalServiceConnectivity validates the connectivity of the service IP +func ValidateExternalServiceConnectivity(serviceIP string, port int) error { + // the default nginx port is 80, skip other ports + if port != 80 { + return nil + } + + err := wait.PollImmediate(pullInterval, pullTimeout, func() (done bool, err error) { + resp, err := http.Get(fmt.Sprintf("http://%s:%d", serviceIP, port)) + if err != nil { + Logf("got error %v, will retry", err) return false, nil } - if targetIP != service.Status.LoadBalancer.Ingress[0].IP == expectSame { - if expectSame { - Logf("still unmatched external IP, retry it in %v seconds", poll) - } else { - Logf("External IP is still %s", targetIP) - } - return false, nil + + if 200 <= resp.StatusCode && resp.StatusCode < 300 { + Logf("succeeded") + return true, nil } - Logf("Exposure successfully") - return true, nil + + Logf("got status code %d", resp.StatusCode) + return false, nil }) + + Logf("validation finished") + return err } // extractSuffix obtains the server domain name suffix