Skip to content

chore: add e2e test for byo public IP #627

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
k8s-ci-robot merged 1 commit into from
May 26, 2021
Merged

chore: add e2e test for byo public IP #627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions site/content/en/topics/tagging-resources.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ the controller manager would parse this configuration and tag the shared resourc

The non-shared resource (public IP) could be tagged by setting `tags` in `azure.json` or service annotation `service.beta.kubernetes.io/azure-pip-tags`. The format of the two is similiar and the tags in the annotation would be considered first when there are conflicts between the configuration file and the annotation.

> The annotation `service.beta.kubernetes.io/azure-pip-tags` only works for managed public IPs. For BYO public IPs, the cloud provider would not apply any tags to them.

When the configuration, file or annotation, is updated, the old ones would be updated if there are conflicts. For example, after updating `{"tags": "a=b,c=d"}` to `{"tags": "a=c,e=f"}`, the new tags would be `a=c,c=d,e=f`.

## Integrating with system tags
Expand Down
107 changes: 85 additions & 22 deletions tests/e2e/network/ensureloadbalancer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package network

import (
"context"
"fmt"
"os"
"strings"
"time"
Expand Down Expand Up @@ -87,6 +88,45 @@ var _ = Describe("Ensure LoadBalancer", func() {
tc = nil
})

It("should support BYO public IP", func() {
By("creating a public IP with tags")
ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4]
pip := defaultPublicIPAddress(ipName)
expectedTags := map[string]*string{
"foo": to.StringPtr("bar"),
}
pip.Tags = expectedTags
pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip)
Expect(err).NotTo(HaveOccurred())
Expect(pip.Tags).To(Equal(expectedTags))
targetIP := to.String(pip.IPAddress)
utils.Logf("created pip with address %s", targetIP)

By("creating a service referencing the public IP")
service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, ports)
service = updateServiceBalanceIP(service, false, targetIP)
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "")
Expect(err).NotTo(HaveOccurred())
Expect(ip).To(Equal(targetIP))

Comment thread
nilo19 marked this conversation as resolved.
Outdated
By("deleting the service")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())

By("test if the pip still exists")
pip, err = utils.WaitGetPIP(tc, ipName)
Expect(err).NotTo(HaveOccurred())

By("test if the tags are changed")
Expect(pip.Tags).To(Equal(expectedTags))

By("cleaning up")
err = utils.DeletePIPWithRetry(tc, ipName, "")
Expect(err).NotTo(HaveOccurred())
})

// Public w/o IP -> Public w/ IP
It("should support assigning to specific IP when updating public service", func() {
annotation := map[string]string{
Expand All @@ -113,7 +153,7 @@ var _ = Describe("Ensure LoadBalancer", func() {
}()

By("Waiting for exposure of the original service without assigned lb IP")
ip1, err := utils.WaitServiceExposure(cs, ns.Name, testServiceName)
ip1, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "")
Expect(err).NotTo(HaveOccurred())

Expect(ip1).NotTo(Equal(targetIP))
Expand All @@ -126,8 +166,9 @@ var _ = Describe("Ensure LoadBalancer", func() {
_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
Expect(err).NotTo(HaveOccurred())

err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, targetIP, true)
ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP)
Expect(err).NotTo(HaveOccurred())
Expect(ip).To(Equal(targetIP))
})

// Internal w/ IP -> Internal w/ IP
Expand All @@ -150,8 +191,9 @@ var _ = Describe("Ensure LoadBalancer", func() {
Expect(err).NotTo(HaveOccurred())
}()
By("Waiting for exposure of internal service with specific IP")
err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, ip1, true)
ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, ip1)
Expect(err).NotTo(HaveOccurred())
Expect(ip).To(Equal(ip1))
list, errList := cs.CoreV1().Events(ns.Name).List(context.TODO(), metav1.ListOptions{})
Expect(errList).NotTo(HaveOccurred())
utils.Logf("Events list:")
Expand All @@ -169,8 +211,9 @@ var _ = Describe("Ensure LoadBalancer", func() {
_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
Expect(err).NotTo(HaveOccurred())

err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, ip2, true)
ip, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, ip2)
Expect(err).NotTo(HaveOccurred())
Expect(ip).To(Equal(ip2))
})

// internal w/o IP -> public w/ IP
Expand Down Expand Up @@ -198,7 +241,7 @@ var _ = Describe("Ensure LoadBalancer", func() {
}()

By("Waiting for exposure of the original service without assigned lb private IP")
ip1, err := utils.WaitServiceExposure(cs, ns.Name, testServiceName)
ip1, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "")
Expect(err).NotTo(HaveOccurred())
Expect(ip1).NotTo(Equal(targetIP))
list, errList := cs.CoreV1().Events(ns.Name).List(context.TODO(), metav1.ListOptions{})
Expand All @@ -216,8 +259,9 @@ var _ = Describe("Ensure LoadBalancer", func() {
_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
Expect(err).NotTo(HaveOccurred())

err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, targetIP, true)
ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP)
Expect(err).NotTo(HaveOccurred())
Expect(ip).To(Equal(targetIP))
})

It("should have no operation since no change in service when update [Slow]", func() {
Expand Down Expand Up @@ -245,7 +289,7 @@ var _ = Describe("Ensure LoadBalancer", func() {
}()

By("Waiting for exposure of the original service with assigned lb private IP")
targetIP, err = utils.WaitServiceExposure(cs, ns.Name, testServiceName)
targetIP, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP)
Expect(err).NotTo(HaveOccurred())

By("Update without changing the service and wait for a while")
Expand All @@ -256,8 +300,29 @@ var _ = Describe("Ensure LoadBalancer", func() {
_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
Expect(err).NotTo(HaveOccurred())

//Wait for 10 minutes, there should return timeout err, since external ip should not change
err = utils.WaitUpdateServiceExposure(cs, ns.Name, testServiceName, targetIP, false /*expectSame*/)
// Wait for 5 minutes, there should return timeout err, since external ip should not change
err = wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) {
service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{})
if err != nil {
if utils.IsRetryableAPIError(err) {
return false, nil
}
return false, err
}

IngressList := service.Status.LoadBalancer.Ingress
if len(IngressList) == 0 {
err = fmt.Errorf("Cannot find Ingress in limited time")
utils.Logf("Fail to get ingress, retry it in %s seconds", 10)
return false, nil
}
if targetIP == service.Status.LoadBalancer.Ingress[0].IP {
utils.Logf("External IP is still %s", targetIP)
return false, nil
}
utils.Logf("succeeded")
return true, nil
})
Expect(err).To(Equal(wait.ErrWaitTimeout))
})

Expand All @@ -271,8 +336,8 @@ var _ = Describe("Ensure LoadBalancer", func() {
service1 = updateServiceBalanceIP(service1, false, targetIP)
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
ip, err := utils.WaitServiceExposure(cs, ns.Name, "service1")
Expect(ip).To(Equal(targetIP))
ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", targetIP)
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service1 in namespace %s with IP %s", ns.Name, ip)

ports2 := []v1.ServicePort{{
Expand All @@ -283,8 +348,8 @@ var _ = Describe("Ensure LoadBalancer", func() {
service2 = updateServiceBalanceIP(service2, false, targetIP)
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
ip, err = utils.WaitServiceExposure(cs, ns.Name, "service2")
Expect(ip).To(Equal(targetIP))
ip, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", targetIP)
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip)

defer func() {
Expand All @@ -302,7 +367,7 @@ var _ = Describe("Ensure LoadBalancer", func() {
service1 := utils.CreateLoadBalancerServiceManifest("service1", nil, labels, ns.Name, ports)
_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
ip, err := utils.WaitServiceExposure(cs, ns.Name, "service1")
ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", "")
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service1 in namespace %s with IP %s", ns.Name, ip)

Expand All @@ -314,17 +379,15 @@ var _ = Describe("Ensure LoadBalancer", func() {
service2 = updateServiceBalanceIP(service2, false, ip)
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
ip2, err := utils.WaitServiceExposure(cs, ns.Name, "service2")
_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip)
Expect(err).NotTo(HaveOccurred())
Expect(ip2).To(Equal(ip))
utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip)

By("Deleting one service and check if the other service works well")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service1", metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
ip3, err := utils.WaitServiceExposure(cs, ns.Name, "service2")
_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip)
Expect(err).NotTo(HaveOccurred())
Expect(ip3).To(Equal(ip))

By("Deleting all services")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service2", metav1.DeleteOptions{})
Expand Down Expand Up @@ -356,7 +419,7 @@ var _ = Describe("Ensure LoadBalancer", func() {
service1 := utils.CreateLoadBalancerServiceManifest("service1", annotation, labels, ns.Name, ports)
_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
ip, err := utils.WaitServiceExposure(cs, ns.Name, "service1")
ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", "")
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service1 in namespace %s with IP %s", ns.Name, ip)

Expand All @@ -368,8 +431,8 @@ var _ = Describe("Ensure LoadBalancer", func() {
service2.Spec.LoadBalancerIP = ip
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
ip2, err := utils.WaitServiceExposure(cs, ns.Name, "service2")
Expect(ip2).To(Equal(ip))
_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip)
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip)

defer func() {
Expand All @@ -394,7 +457,7 @@ var _ = Describe("Ensure LoadBalancer", func() {
service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, ports)
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
publicIP, err := utils.WaitServiceExposure(cs, ns.Name, testServiceName)
publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "")
Expect(err).NotTo(HaveOccurred())

By("Checking the initial node number in the LB backend pool")
Expand Down
15 changes: 12 additions & 3 deletions tests/e2e/network/network_security_group.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
"github.com/Azure/go-autorest/autorest/to"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
Expand Down Expand Up @@ -141,7 +142,15 @@ var _ = Describe("Network security group", func() {
annotation := map[string]string{
consts.ServiceAnnotationAllowedServiceTag: "AzureCloud",
}
_ = createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
utils.Logf("Creating service " + serviceName + " in namespace " + ns.Name)
service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports)
_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name)

By("Waiting for the service to be exposed")
_, err = utils.WaitServiceExposure(cs, ns.Name, serviceName, "")
Expect(err).NotTo(HaveOccurred())

By("Validating if the corresponding IP prefix existing in nsg")
nsgs, err := tc.GetClusterSecurityGroups()
Expand Down Expand Up @@ -177,7 +186,7 @@ var _ = Describe("Network security group", func() {
Expect(err).NotTo(HaveOccurred())

By("Waiting for the service to expose")
internalIP, err := utils.WaitServiceExposure(cs, ns.Name, serviceName)
internalIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "")
Expect(err).NotTo(HaveOccurred())

By("Checking if there is a deny_all rule")
Expand All @@ -196,7 +205,7 @@ var _ = Describe("Network security group", func() {
Expect(err).NotTo(HaveOccurred())

By("Waiting for the service to expose")
internalIP, err = utils.WaitServiceExposure(cs, ns.Name, serviceName)
internalIP, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "")
Expect(err).NotTo(HaveOccurred())

By("Checking if there is a LoadBalancerSourceRanges rule")
Expand Down
Loading