Skip to content

acceptance: Add retry to resource deletion and wait for more for ELB deletion to propagate #1368

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ishustava merged 1 commit into from
Jul 25, 2022
Merged

acceptance: Add retry to resource deletion and wait for more for ELB deletion to propagate #1368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 64 additions & 63 deletions acceptance/framework/consul/helm_cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,87 +123,88 @@ func (h *HelmCluster) Destroy(t *testing.T) {
// always idempotently clean up resources in the cluster.
_ = helm.DeleteE(t, h.helmOptions, h.releaseName, false)

// Force delete any pods that have h.releaseName in their name because sometimes
// graceful termination takes a long time and since this is an uninstall
// we don't care that they're stopped gracefully.
pods, err := h.kubernetesClient.CoreV1().Pods(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
for _, pod := range pods.Items {
if strings.Contains(pod.Name, h.releaseName) {
var gracePeriod int64 = 0
err := h.kubernetesClient.CoreV1().Pods(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod})
if !errors.IsNotFound(err) {
require.NoError(t, err)
// Retry because sometimes certain resources (like PVC) take time to delete
// in cloud providers.
retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 600}, t, func(r *retry.R) {
// Force delete any pods that have h.releaseName in their name because sometimes
// graceful termination takes a long time and since this is an uninstall
// we don't care that they're stopped gracefully.
pods, err := h.kubernetesClient.CoreV1().Pods(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
for _, pod := range pods.Items {
if strings.Contains(pod.Name, h.releaseName) {
var gracePeriod int64 = 0
err := h.kubernetesClient.CoreV1().Pods(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod})
if !errors.IsNotFound(err) {
require.NoError(t, err)
}
}
}
}

// Delete PVCs.
err = h.kubernetesClient.CoreV1().PersistentVolumeClaims(h.helmOptions.KubectlOptions.Namespace).DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
// Delete PVCs.
err = h.kubernetesClient.CoreV1().PersistentVolumeClaims(h.helmOptions.KubectlOptions.Namespace).DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)

// Delete any serviceaccounts that have h.releaseName in their name.
sas, err := h.kubernetesClient.CoreV1().ServiceAccounts(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
for _, sa := range sas.Items {
if strings.Contains(sa.Name, h.releaseName) {
err := h.kubernetesClient.CoreV1().ServiceAccounts(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), sa.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
require.NoError(t, err)
// Delete any serviceaccounts that have h.releaseName in their name.
sas, err := h.kubernetesClient.CoreV1().ServiceAccounts(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
for _, sa := range sas.Items {
if strings.Contains(sa.Name, h.releaseName) {
err := h.kubernetesClient.CoreV1().ServiceAccounts(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), sa.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
require.NoError(t, err)
}
}
}
}

// Delete any roles that have h.releaseName in their name.
roles, err := h.kubernetesClient.RbacV1().Roles(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
for _, role := range roles.Items {
if strings.Contains(role.Name, h.releaseName) {
err := h.kubernetesClient.RbacV1().Roles(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), role.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
require.NoError(t, err)
// Delete any roles that have h.releaseName in their name.
roles, err := h.kubernetesClient.RbacV1().Roles(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
for _, role := range roles.Items {
if strings.Contains(role.Name, h.releaseName) {
err := h.kubernetesClient.RbacV1().Roles(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), role.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
require.NoError(t, err)
}
}
}
}

// Delete any rolebindings that have h.releaseName in their name.
roleBindings, err := h.kubernetesClient.RbacV1().RoleBindings(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
for _, roleBinding := range roleBindings.Items {
if strings.Contains(roleBinding.Name, h.releaseName) {
err := h.kubernetesClient.RbacV1().RoleBindings(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), roleBinding.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
require.NoError(t, err)
// Delete any rolebindings that have h.releaseName in their name.
roleBindings, err := h.kubernetesClient.RbacV1().RoleBindings(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
for _, roleBinding := range roleBindings.Items {
if strings.Contains(roleBinding.Name, h.releaseName) {
err := h.kubernetesClient.RbacV1().RoleBindings(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), roleBinding.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
require.NoError(t, err)
}
}
}
}

// Delete any secrets that have h.releaseName in their name.
secrets, err := h.kubernetesClient.CoreV1().Secrets(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{})
require.NoError(t, err)
for _, secret := range secrets.Items {
if strings.Contains(secret.Name, h.releaseName) {
err := h.kubernetesClient.CoreV1().Secrets(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), secret.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
require.NoError(t, err)
// Delete any secrets that have h.releaseName in their name.
secrets, err := h.kubernetesClient.CoreV1().Secrets(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{})
require.NoError(t, err)
for _, secret := range secrets.Items {
if strings.Contains(secret.Name, h.releaseName) {
err := h.kubernetesClient.CoreV1().Secrets(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), secret.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
require.NoError(t, err)
}
}
}
}

// Delete any jobs that have h.releaseName in their name.
jobs, err := h.kubernetesClient.BatchV1().Jobs(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
for _, job := range jobs.Items {
if strings.Contains(job.Name, h.releaseName) {
err := h.kubernetesClient.BatchV1().Jobs(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), job.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
require.NoError(t, err)
// Delete any jobs that have h.releaseName in their name.
jobs, err := h.kubernetesClient.BatchV1().Jobs(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(t, err)
for _, job := range jobs.Items {
if strings.Contains(job.Name, h.releaseName) {
err := h.kubernetesClient.BatchV1().Jobs(h.helmOptions.KubectlOptions.Namespace).Delete(context.Background(), job.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
require.NoError(t, err)
}
}
}
}
// Retry because sometimes certain resources (like PVC) take time to delete
// in cloud providers.
retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 600}, t, func(r *retry.R) {

// Verify all Consul Pods are deleted.
pods, err = h.kubernetesClient.CoreV1().Pods(h.helmOptions.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "release=" + h.releaseName})
require.NoError(r, err)
Expand Down
2 changes: 1 addition & 1 deletion hack/aws-acceptance-test-cleanup/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ func realMain(ctx context.Context) error {
return err
}
// Allow time for ELB deletion to propagate so that we can detach the internet gateway.
time.Sleep(1 * time.Second)
time.Sleep(10 * time.Second)
fmt.Printf("ELB: Destroyed [id=%s]\n", *elbDescrip.LoadBalancerName)
}

Expand Down