From 0de28539fe6e0e84bd59e28051b8b9ca5d1fe44c Mon Sep 17 00:00:00 2001 From: Huy Mai Date: Mon, 8 Apr 2024 15:29:46 +0300 Subject: [PATCH 1/4] Ensure E2E cleanup Signed-off-by: Huy Mai --- test/e2e/shared/openstack.go | 26 +++++++++++++++++ test/e2e/suites/e2e/e2e_suite_test.go | 40 ++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/test/e2e/shared/openstack.go b/test/e2e/shared/openstack.go index 2fd5f95d56..a11593beab 100644 --- a/test/e2e/shared/openstack.go +++ b/test/e2e/shared/openstack.go @@ -39,6 +39,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors" "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" + "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules" @@ -865,3 +866,28 @@ func GetFlavorFromName(e2eCtx *E2EContext, name string) (*flavors.Flavor, error) return flavors.Get(computeClient, flavorID).Extract() } + +func DumpOpenStackLoadBalancers(e2eCtx *E2EContext, filter loadbalancers.ListOpts) ([]loadbalancers.LoadBalancer, error) { + providerClient, clientOpts, _, err := GetTenantProviderClient(e2eCtx) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "error creating provider client: %s\n", err) + return nil, err + } + + loadBalancerClient, err := openstack.NewLoadBalancerV2(providerClient, gophercloud.EndpointOpts{ + Region: clientOpts.RegionName, + }) + if err != nil { + return nil, fmt.Errorf("error creating network client: %s", err) + } + + allPages, err := loadbalancers.List(loadBalancerClient, filter).AllPages() + if err != nil { + return nil, fmt.Errorf("error getting load balancers: %s", err) + } + loadBalancersList, err := loadbalancers.ExtractLoadBalancers(allPages) + if err != nil { + return nil, fmt.Errorf("error getting load balancers: %s", err) + } + return loadBalancersList, nil +} diff --git a/test/e2e/suites/e2e/e2e_suite_test.go b/test/e2e/suites/e2e/e2e_suite_test.go index 177c5ee2cc..460b156e55 100644 --- a/test/e2e/suites/e2e/e2e_suite_test.go +++ b/test/e2e/suites/e2e/e2e_suite_test.go @@ -23,6 +23,11 @@ import ( "os" "testing" + "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes" + "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" + "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups" + "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/klog/v2" @@ -31,7 +36,15 @@ import ( "sigs.k8s.io/cluster-api-provider-openstack/test/e2e/shared" ) -var e2eCtx *shared.E2EContext +var ( + e2eCtx *shared.E2EContext + initialServers []servers.Server + initialNetworks []networks.Network + initialSecurityGroups []groups.SecGroup + initialLoadBalancers []loadbalancers.LoadBalancer + initialVolumes []volumes.Volume + err error +) func init() { e2eCtx = shared.NewE2EContext() @@ -54,6 +67,16 @@ var _ = SynchronizedBeforeSuite(func() []byte { data := shared.Node1BeforeSuite(e2eCtx) return data }, func(data []byte) { + initialServers, err = shared.DumpOpenStackServers(e2eCtx, servers.ListOpts{}) + Expect(err).NotTo(HaveOccurred()) + initialNetworks, err = shared.DumpOpenStackNetworks(e2eCtx, networks.ListOpts{}) + Expect(err).NotTo(HaveOccurred()) + initialSecurityGroups, err = shared.DumpOpenStackSecurityGroups(e2eCtx, groups.ListOpts{}) + Expect(err).NotTo(HaveOccurred()) + initialLoadBalancers, err = shared.DumpOpenStackLoadBalancers(e2eCtx, loadbalancers.ListOpts{}) + Expect(err).NotTo(HaveOccurred()) + initialVolumes, err = shared.DumpOpenStackVolumes(e2eCtx, volumes.ListOpts{}) + Expect(err).NotTo(HaveOccurred()) shared.AllNodesBeforeSuite(e2eCtx, data) }) @@ -61,4 +84,19 @@ var _ = SynchronizedAfterSuite(func() { shared.AllNodesAfterSuite(e2eCtx) }, func() { shared.Node1AfterSuite(e2eCtx) + endServers, err := shared.DumpOpenStackServers(e2eCtx, servers.ListOpts{}) + Expect(err).NotTo(HaveOccurred()) + Expect(endServers).To(Equal(initialServers)) + endNetworks, err := shared.DumpOpenStackNetworks(e2eCtx, networks.ListOpts{}) + Expect(err).NotTo(HaveOccurred()) + Expect(endNetworks).To(Equal(initialNetworks)) + endSecurityGroups, err := shared.DumpOpenStackSecurityGroups(e2eCtx, groups.ListOpts{}) + Expect(err).NotTo(HaveOccurred()) + Expect(endSecurityGroups).To(Equal(initialSecurityGroups)) + endLoadBalancers, err := shared.DumpOpenStackLoadBalancers(e2eCtx, loadbalancers.ListOpts{}) + Expect(err).NotTo(HaveOccurred()) + Expect(endLoadBalancers).To(Equal(initialLoadBalancers)) + endVolumes, err := shared.DumpOpenStackVolumes(e2eCtx, volumes.ListOpts{}) + Expect(err).NotTo(HaveOccurred()) + Expect(endVolumes).To(Equal(initialVolumes)) }) From 6e200281c46fd8ab182f9e46fa062919c0e70e8a Mon Sep 17 00:00:00 2001 From: Matthew Booth Date: Tue, 9 Apr 2024 17:57:26 +0100 Subject: [PATCH 2/4] Initialize e2eCtx before dumping OpenStack resources --- test/e2e/suites/e2e/e2e_suite_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/e2e/suites/e2e/e2e_suite_test.go b/test/e2e/suites/e2e/e2e_suite_test.go index 460b156e55..4e34725089 100644 --- a/test/e2e/suites/e2e/e2e_suite_test.go +++ b/test/e2e/suites/e2e/e2e_suite_test.go @@ -67,6 +67,8 @@ var _ = SynchronizedBeforeSuite(func() []byte { data := shared.Node1BeforeSuite(e2eCtx) return data }, func(data []byte) { + shared.AllNodesBeforeSuite(e2eCtx, data) + initialServers, err = shared.DumpOpenStackServers(e2eCtx, servers.ListOpts{}) Expect(err).NotTo(HaveOccurred()) initialNetworks, err = shared.DumpOpenStackNetworks(e2eCtx, networks.ListOpts{}) @@ -77,13 +79,11 @@ var _ = SynchronizedBeforeSuite(func() []byte { Expect(err).NotTo(HaveOccurred()) initialVolumes, err = shared.DumpOpenStackVolumes(e2eCtx, volumes.ListOpts{}) Expect(err).NotTo(HaveOccurred()) - shared.AllNodesBeforeSuite(e2eCtx, data) }) var _ = SynchronizedAfterSuite(func() { shared.AllNodesAfterSuite(e2eCtx) }, func() { - shared.Node1AfterSuite(e2eCtx) endServers, err := shared.DumpOpenStackServers(e2eCtx, servers.ListOpts{}) Expect(err).NotTo(HaveOccurred()) Expect(endServers).To(Equal(initialServers)) @@ -99,4 +99,6 @@ var _ = SynchronizedAfterSuite(func() { endVolumes, err := shared.DumpOpenStackVolumes(e2eCtx, volumes.ListOpts{}) Expect(err).NotTo(HaveOccurred()) Expect(endVolumes).To(Equal(initialVolumes)) + + shared.Node1AfterSuite(e2eCtx) }) From 7eef0289652727d0143c32b6c9daa699c49bc0be Mon Sep 17 00:00:00 2001 From: Matthew Booth Date: Tue, 9 Apr 2024 21:03:04 +0100 Subject: [PATCH 3/4] Add gomega ID matcher for gophercloud objects --- test/e2e/suites/e2e/e2e_suite_test.go | 10 ++--- test/e2e/suites/e2e/idmatcher_test.go | 59 +++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 test/e2e/suites/e2e/idmatcher_test.go diff --git a/test/e2e/suites/e2e/e2e_suite_test.go b/test/e2e/suites/e2e/e2e_suite_test.go index 4e34725089..5cd2e7805b 100644 --- a/test/e2e/suites/e2e/e2e_suite_test.go +++ b/test/e2e/suites/e2e/e2e_suite_test.go @@ -86,19 +86,19 @@ var _ = SynchronizedAfterSuite(func() { }, func() { endServers, err := shared.DumpOpenStackServers(e2eCtx, servers.ListOpts{}) Expect(err).NotTo(HaveOccurred()) - Expect(endServers).To(Equal(initialServers)) + Expect(endServers).To(ConsistOfIDs(initialServers)) endNetworks, err := shared.DumpOpenStackNetworks(e2eCtx, networks.ListOpts{}) Expect(err).NotTo(HaveOccurred()) - Expect(endNetworks).To(Equal(initialNetworks)) + Expect(endNetworks).To(ConsistOfIDs(initialNetworks)) endSecurityGroups, err := shared.DumpOpenStackSecurityGroups(e2eCtx, groups.ListOpts{}) Expect(err).NotTo(HaveOccurred()) - Expect(endSecurityGroups).To(Equal(initialSecurityGroups)) + Expect(endSecurityGroups).To(ConsistOfIDs(initialSecurityGroups)) endLoadBalancers, err := shared.DumpOpenStackLoadBalancers(e2eCtx, loadbalancers.ListOpts{}) Expect(err).NotTo(HaveOccurred()) - Expect(endLoadBalancers).To(Equal(initialLoadBalancers)) + Expect(endLoadBalancers).To(ConsistOfIDs(initialLoadBalancers)) endVolumes, err := shared.DumpOpenStackVolumes(e2eCtx, volumes.ListOpts{}) Expect(err).NotTo(HaveOccurred()) - Expect(endVolumes).To(Equal(initialVolumes)) + Expect(endVolumes).To(ConsistOfIDs(initialVolumes)) shared.Node1AfterSuite(e2eCtx) }) diff --git a/test/e2e/suites/e2e/idmatcher_test.go b/test/e2e/suites/e2e/idmatcher_test.go new file mode 100644 index 0000000000..146d6c3e53 --- /dev/null +++ b/test/e2e/suites/e2e/idmatcher_test.go @@ -0,0 +1,59 @@ +package e2e + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/gstruct" + "github.com/onsi/gomega/types" +) + +type idMatcher struct { + expected interface{} +} + +var ( + _ types.GomegaMatcher = &idMatcher{} + _ format.GomegaStringer = &idMatcher{} +) + +func (m *idMatcher) Match(actual interface{}) (bool, error) { + v := reflect.ValueOf(m.expected) + id := v.FieldByName("ID").String() + + matcher := &gstruct.FieldsMatcher{ + Fields: gstruct.Fields{ + "ID": gomega.Equal(id), + }, + IgnoreExtras: true, + } + + return matcher.Match(actual) +} + +func (m *idMatcher) FailureMessage(actual interface{}) string { + return fmt.Sprintf("Expected:\n%s\nto have the same ID as:\n%s", format.Object(actual, 1), format.Object(m.expected, 1)) +} + +func (m *idMatcher) NegatedFailureMessage(actual interface{}) string { + return fmt.Sprintf("Expected:\n%s\nnot to have the same ID as:\n%s", format.Object(actual, 1), format.Object(m.expected, 1)) +} + +func (m *idMatcher) GomegaString() string { + return fmt.Sprintf("ID match for:\n%s", format.Object(m.expected, 1)) +} + +func IDOf(expected interface{}) types.GomegaMatcher { + return &idMatcher{expected: expected} +} + +func ConsistOfIDs[T any](expected []T) types.GomegaMatcher { + matchers := make([]types.GomegaMatcher, len(expected)) + for i := range expected { + matchers[i] = IDOf(expected[i]) + } + + return gomega.ConsistOf(matchers) +} From 7863a7a2b5a3fa30d66bb968ea58649491830e8d Mon Sep 17 00:00:00 2001 From: Matthew Booth Date: Tue, 9 Apr 2024 23:29:59 +0100 Subject: [PATCH 4/4] Display all resource cleanup errors --- test/e2e/suites/e2e/e2e_suite_test.go | 52 +++++++++++++++++++-------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/test/e2e/suites/e2e/e2e_suite_test.go b/test/e2e/suites/e2e/e2e_suite_test.go index 5cd2e7805b..ae02892a69 100644 --- a/test/e2e/suites/e2e/e2e_suite_test.go +++ b/test/e2e/suites/e2e/e2e_suite_test.go @@ -31,6 +31,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/klog/v2" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/cluster-api-provider-openstack/test/e2e/shared" @@ -81,24 +82,45 @@ var _ = SynchronizedBeforeSuite(func() []byte { Expect(err).NotTo(HaveOccurred()) }) +func CheckResourceCleanup[T any, L any](f func(*shared.E2EContext, L) ([]T, error), l L, initialResources []T) *string { + endResources, err := f(e2eCtx, l) + + if err != nil { + return pointer.String(err.Error()) + } + + matcher := ConsistOfIDs(initialResources) + success, err := matcher.Match(endResources) + if err != nil { + return pointer.String(err.Error()) + } + if !success { + return pointer.String(matcher.FailureMessage(endResources)) + } + + return nil +} + var _ = SynchronizedAfterSuite(func() { shared.AllNodesAfterSuite(e2eCtx) }, func() { - endServers, err := shared.DumpOpenStackServers(e2eCtx, servers.ListOpts{}) - Expect(err).NotTo(HaveOccurred()) - Expect(endServers).To(ConsistOfIDs(initialServers)) - endNetworks, err := shared.DumpOpenStackNetworks(e2eCtx, networks.ListOpts{}) - Expect(err).NotTo(HaveOccurred()) - Expect(endNetworks).To(ConsistOfIDs(initialNetworks)) - endSecurityGroups, err := shared.DumpOpenStackSecurityGroups(e2eCtx, groups.ListOpts{}) - Expect(err).NotTo(HaveOccurred()) - Expect(endSecurityGroups).To(ConsistOfIDs(initialSecurityGroups)) - endLoadBalancers, err := shared.DumpOpenStackLoadBalancers(e2eCtx, loadbalancers.ListOpts{}) - Expect(err).NotTo(HaveOccurred()) - Expect(endLoadBalancers).To(ConsistOfIDs(initialLoadBalancers)) - endVolumes, err := shared.DumpOpenStackVolumes(e2eCtx, volumes.ListOpts{}) - Expect(err).NotTo(HaveOccurred()) - Expect(endVolumes).To(ConsistOfIDs(initialVolumes)) + failed := false + for _, error := range []*string{ + CheckResourceCleanup(shared.DumpOpenStackServers, servers.ListOpts{}, initialServers), + CheckResourceCleanup(shared.DumpOpenStackNetworks, networks.ListOpts{}, initialNetworks), + CheckResourceCleanup(shared.DumpOpenStackSecurityGroups, groups.ListOpts{}, initialSecurityGroups), + CheckResourceCleanup(shared.DumpOpenStackLoadBalancers, loadbalancers.ListOpts{}, initialLoadBalancers), + CheckResourceCleanup(shared.DumpOpenStackVolumes, volumes.ListOpts{}, initialVolumes), + } { + if error != nil { + GinkgoWriter.Println(*error) + failed = true + } + } shared.Node1AfterSuite(e2eCtx) + + if failed { + Fail("Not all resources were cleaned up") + } })