From fe8eab15adb72a7a5ec6e387c8b00c52dd232440 Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Mon, 25 Jul 2022 09:44:37 -0600 Subject: [PATCH] acceptance: Add retry to resource deletion and wait for more for ELB deletion to propagate --- acceptance/framework/consul/helm_cluster.go | 127 ++++++++++---------- hack/aws-acceptance-test-cleanup/main.go | 2 +- 2 files changed, 65 insertions(+), 64 deletions(-) diff --git a/acceptance/framework/consul/helm_cluster.go b/acceptance/framework/consul/helm_cluster.go index a6530c205b..bcd07ecf39 100644 --- a/acceptance/framework/consul/helm_cluster.go +++ b/acceptance/framework/consul/helm_cluster.go @@ -123,87 +123,88 @@ func (h *HelmCluster) Destroy(t *testing.T) { // always idempotently clean up resources in the cluster. _ = helm.DeleteE(t, h.helmOptions, h.releaseName, false) - // Force delete any pods that have h.releaseName in their name because sometimes - // graceful termination takes a long time and since this is an uninstall - // we don't care that they're stopped gracefully. - pods, err := h.kubernetesClient.CoreV1().Pods(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) - require.NoError(t, err) - for _, pod := range pods.Items { - if strings.Contains(pod.Name, h.releaseName) { - var gracePeriod int64 = 0 - err := h.kubernetesClient.CoreV1().Pods(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}) - if !errors.IsNotFound(err) { - require.NoError(t, err) + // Retry because sometimes certain resources (like PVC) take time to delete + // in cloud providers. + retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 600}, t, func(r *retry.R) { + // Force delete any pods that have h.releaseName in their name because sometimes + // graceful termination takes a long time and since this is an uninstall + // we don't care that they're stopped gracefully. + pods, err := h.kubernetesClient.CoreV1().Pods(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) + require.NoError(t, err) + for _, pod := range pods.Items { + if strings.Contains(pod.Name, h.releaseName) { + var gracePeriod int64 = 0 + err := h.kubernetesClient.CoreV1().Pods(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}) + if !errors.IsNotFound(err) { + require.NoError(t, err) + } } } - } - // Delete PVCs. - err = h.kubernetesClient.CoreV1().PersistentVolumeClaims(h.helmOptions.KubectlOptions.Namespace).DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) - require.NoError(t, err) + // Delete PVCs. + err = h.kubernetesClient.CoreV1().PersistentVolumeClaims(h.helmOptions.KubectlOptions.Namespace).DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) + require.NoError(t, err) - // Delete any serviceaccounts that have h.releaseName in their name. - sas, err := h.kubernetesClient.CoreV1().ServiceAccounts(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) - require.NoError(t, err) - for _, sa := range sas.Items { - if strings.Contains(sa.Name, h.releaseName) { - err := h.kubernetesClient.CoreV1().ServiceAccounts(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), sa.Name, metav1.DeleteOptions{}) - if !errors.IsNotFound(err) { - require.NoError(t, err) + // Delete any serviceaccounts that have h.releaseName in their name. + sas, err := h.kubernetesClient.CoreV1().ServiceAccounts(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) + require.NoError(t, err) + for _, sa := range sas.Items { + if strings.Contains(sa.Name, h.releaseName) { + err := h.kubernetesClient.CoreV1().ServiceAccounts(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), sa.Name, metav1.DeleteOptions{}) + if !errors.IsNotFound(err) { + require.NoError(t, err) + } } } - } - // Delete any roles that have h.releaseName in their name. - roles, err := h.kubernetesClient.RbacV1().Roles(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) - require.NoError(t, err) - for _, role := range roles.Items { - if strings.Contains(role.Name, h.releaseName) { - err := h.kubernetesClient.RbacV1().Roles(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), role.Name, metav1.DeleteOptions{}) - if !errors.IsNotFound(err) { - require.NoError(t, err) + // Delete any roles that have h.releaseName in their name. + roles, err := h.kubernetesClient.RbacV1().Roles(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) + require.NoError(t, err) + for _, role := range roles.Items { + if strings.Contains(role.Name, h.releaseName) { + err := h.kubernetesClient.RbacV1().Roles(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), role.Name, metav1.DeleteOptions{}) + if !errors.IsNotFound(err) { + require.NoError(t, err) + } } } - } - // Delete any rolebindings that have h.releaseName in their name. - roleBindings, err := h.kubernetesClient.RbacV1().RoleBindings(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) - require.NoError(t, err) - for _, roleBinding := range roleBindings.Items { - if strings.Contains(roleBinding.Name, h.releaseName) { - err := h.kubernetesClient.RbacV1().RoleBindings(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), roleBinding.Name, metav1.DeleteOptions{}) - if !errors.IsNotFound(err) { - require.NoError(t, err) + // Delete any rolebindings that have h.releaseName in their name. + roleBindings, err := h.kubernetesClient.RbacV1().RoleBindings(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) + require.NoError(t, err) + for _, roleBinding := range roleBindings.Items { + if strings.Contains(roleBinding.Name, h.releaseName) { + err := h.kubernetesClient.RbacV1().RoleBindings(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), roleBinding.Name, metav1.DeleteOptions{}) + if !errors.IsNotFound(err) { + require.NoError(t, err) + } } } - } - // Delete any secrets that have h.releaseName in their name. - secrets, err := h.kubernetesClient.CoreV1().Secrets(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{}) - require.NoError(t, err) - for _, secret := range secrets.Items { - if strings.Contains(secret.Name, h.releaseName) { - err := h.kubernetesClient.CoreV1().Secrets(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), secret.Name, metav1.DeleteOptions{}) - if !errors.IsNotFound(err) { - require.NoError(t, err) + // Delete any secrets that have h.releaseName in their name. + secrets, err := h.kubernetesClient.CoreV1().Secrets(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{}) + require.NoError(t, err) + for _, secret := range secrets.Items { + if strings.Contains(secret.Name, h.releaseName) { + err := h.kubernetesClient.CoreV1().Secrets(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), secret.Name, metav1.DeleteOptions{}) + if !errors.IsNotFound(err) { + require.NoError(t, err) + } } } - } - // Delete any jobs that have h.releaseName in their name. - jobs, err := h.kubernetesClient.BatchV1().Jobs(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) - require.NoError(t, err) - for _, job := range jobs.Items { - if strings.Contains(job.Name, h.releaseName) { - err := h.kubernetesClient.BatchV1().Jobs(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), job.Name, metav1.DeleteOptions{}) - if !errors.IsNotFound(err) { - require.NoError(t, err) + // Delete any jobs that have h.releaseName in their name. + jobs, err := h.kubernetesClient.BatchV1().Jobs(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) + require.NoError(t, err) + for _, job := range jobs.Items { + if strings.Contains(job.Name, h.releaseName) { + err := h.kubernetesClient.BatchV1().Jobs(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), job.Name, metav1.DeleteOptions{}) + if !errors.IsNotFound(err) { + require.NoError(t, err) + } } } - } - // Retry because sometimes certain resources (like PVC) take time to delete - // in cloud providers. - retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 600}, t, func(r *retry.R) { + // Verify all Consul Pods are deleted. pods, err = h.kubernetesClient.CoreV1().Pods(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName}) require.NoError(r, err) diff --git a/hack/aws-acceptance-test-cleanup/main.go b/hack/aws-acceptance-test-cleanup/main.go index e24fd692ce..3c121dfa11 100644 --- a/hack/aws-acceptance-test-cleanup/main.go +++ b/hack/aws-acceptance-test-cleanup/main.go @@ -364,7 +364,7 @@ func realMain(ctx context.Context) error { return err } // Allow time for ELB deletion to propagate so that we can detach the internet gateway. - time.Sleep(1 * time.Second) + time.Sleep(10 * time.Second) fmt.Printf("ELB: Destroyed [id=%s]\n", *elbDescrip.LoadBalancerName) }