diff --git a/cmd/e2e-test/finalizer_test.go b/cmd/e2e-test/finalizer_test.go new file mode 100644 index 0000000000..51a3417b57 --- /dev/null +++ b/cmd/e2e-test/finalizer_test.go @@ -0,0 +1,160 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "testing" + + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-gce/pkg/e2e" + "k8s.io/ingress-gce/pkg/fuzz" + "k8s.io/ingress-gce/pkg/fuzz/features" + "k8s.io/ingress-gce/pkg/utils" +) + +func TestFinalizer(t *testing.T) { + t.Parallel() + ctx := context.Background() + port80 := intstr.FromInt(80) + ing := fuzz.NewIngressBuilder("", "ingress-1", ""). + AddPath("foo.com", "/", "service-1", port80). + SetIngressClass("gce"). + Build() + + Framework.RunWithSandbox("finalizer", t, func(t *testing.T, s *e2e.Sandbox) { + _, err := e2e.CreateEchoService(s, "service-1", nil) + if err != nil { + t.Fatalf("error creating echo service: %v", err) + } + t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") + + crud := e2e.IngressCRUD{C: Framework.Clientset} + ing.Namespace = s.Namespace + if _, err := crud.Create(ing); err != nil { + t.Fatalf("error creating Ingress spec: %v", err) + } + t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) + + ing, err := e2e.WaitForIngress(s, ing, nil) + if err != nil { + t.Fatalf("error waiting for Ingress to stabilize: %v", err) + } + t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name) + + // Perform whitebox testing. + if len(ing.Status.LoadBalancer.Ingress) < 1 { + t.Fatalf("Ingress does not have an IP: %+v", ing.Status) + } + + ingFinalizers := ing.GetFinalizers() + if len(ingFinalizers) != 1 || ingFinalizers[0] != utils.FinalizerKey { + t.Fatalf("GetFinalizers() = %+v, want [%s]", ingFinalizers, utils.FinalizerKey) + } + + vip := ing.Status.LoadBalancer.Ingress[0].IP + t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip) + gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, vip, fuzz.FeatureValidators(features.All)) + if err != nil { + t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err) + } + + if err = e2e.CheckGCLB(gclb, 1, 2); err != nil { + t.Error(err) + } + + deleteOptions := &fuzz.GCLBDeleteOptions{ + SkipDefaultBackend: true, + } + if err := e2e.WaitForIngressDeletion(ctx, gclb, s, ing, deleteOptions); err != nil { + t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", ing.Name, err) + } + }) +} + +func TestFinalizerIngressClassChange(t *testing.T) { + t.Parallel() + ctx := context.Background() + port80 := intstr.FromInt(80) + ing := fuzz.NewIngressBuilder("", "ingress-1", ""). + AddPath("foo.com", "/", "service-1", port80). + SetIngressClass("gce"). + Build() + + Framework.RunWithSandbox("finalizer-ingress-class-change", t, func(t *testing.T, s *e2e.Sandbox) { + _, err := e2e.CreateEchoService(s, "service-1", nil) + if err != nil { + t.Fatalf("error creating echo service: %v", err) + } + t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") + + crud := e2e.IngressCRUD{C: Framework.Clientset} + ing.Namespace = s.Namespace + if _, err := crud.Create(ing); err != nil { + t.Fatalf("error creating Ingress spec: %v", err) + } + t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) + + ing, err := e2e.WaitForIngress(s, ing, nil) + if err != nil { + t.Fatalf("error waiting for Ingress to stabilize: %v", err) + } + t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name) + + ingFinalizers := ing.GetFinalizers() + if len(ingFinalizers) != 1 || ingFinalizers[0] != utils.FinalizerKey { + t.Fatalf("GetFinalizers() = %+v, want [%q]", ingFinalizers, utils.FinalizerKey) + } + + // Perform whitebox testing. + if len(ing.Status.LoadBalancer.Ingress) < 1 { + t.Fatalf("Ingress does not have an IP: %+v", ing.Status) + } + + vip := ing.Status.LoadBalancer.Ingress[0].IP + t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip) + gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, vip, fuzz.FeatureValidators(features.All)) + if err != nil { + t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err) + } + + if err = e2e.CheckGCLB(gclb, 1, 2); err != nil { + t.Error(err) + } + + // Change Ingress class + newIngClass := "nginx" + ing = fuzz.NewIngressBuilderFromExisting(ing).SetIngressClass(newIngClass).Build() + + if _, err := crud.Update(ing); err != nil { + t.Fatalf("error updating Ingress spec: %v", err) + } + t.Logf("Ingress (%s/%s) class changed to %s", s.Namespace, ing.Name, newIngClass) + + deleteOptions := &fuzz.GCLBDeleteOptions{ + SkipDefaultBackend: true, + } + if err := e2e.WaitForFinalizerDeletion(ctx, gclb, s, ing.Name, deleteOptions); err != nil { + t.Errorf("e2e.WaitForFinalizerDeletion(...) = %v, want nil", err) + } + t.Logf("Finalizer for Ingress (%s/%s) deleted", s.Namespace, ing.Name) + + if err := e2e.WaitForIngressDeletion(ctx, gclb, s, ing, deleteOptions); err != nil { + t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", ing.Name, err) + } + }) +} diff --git a/pkg/e2e/helpers.go b/pkg/e2e/helpers.go index db013cdbc2..6e64705e1f 100644 --- a/pkg/e2e/helpers.go +++ b/pkg/e2e/helpers.go @@ -106,6 +106,31 @@ func WaitForIngressDeletion(ctx context.Context, g *fuzz.GCLB, s *Sandbox, ing * return nil } +// WaitForFinalizerDeletion waits for gclb resources to be deleted and +// the finalizer attached to the Ingress resource to be removed. +func WaitForFinalizerDeletion(ctx context.Context, g *fuzz.GCLB, s *Sandbox, ingName string, options *fuzz.GCLBDeleteOptions) error { + klog.Infof("Waiting for GCLB resources to be deleted (%s/%s), IngressDeletionOptions=%+v", s.Namespace, ingName, options) + if err := WaitForGCLBDeletion(ctx, s.f.Cloud, g, options); err != nil { + return fmt.Errorf("WaitForGCLBDeletion(...) = %v, want nil", err) + } + klog.Infof("GCLB resources deleted (%s/%s)", s.Namespace, ingName) + + crud := IngressCRUD{s.f.Clientset} + klog.Infof("Waiting for Finalizer to be removed for Ingress %s/%s", s.Namespace, ingName) + return wait.Poll(k8sApiPoolInterval, k8sApiPollTimeout, func() (bool, error) { + ing, err := crud.Get(s.Namespace, ingName) + if err != nil { + klog.Infof("WaitForFinalizerDeletion(%s/%s) = Error retrieving Ingress: %v", s.Namespace, ing.Name, err) + return false, nil + } + if len(ing.GetFinalizers()) != 0 { + klog.Infof("WaitForFinalizerDeletion(%s/%s) = %v", s.Namespace, ing.Name, ing.GetFinalizers()) + return false, nil + } + return true, nil + }) +} + // WaitForGCLBDeletion waits for the resources associated with the GLBC to be // deleted. func WaitForGCLBDeletion(ctx context.Context, c cloud.Cloud, g *fuzz.GCLB, options *fuzz.GCLBDeleteOptions) error { diff --git a/pkg/e2e/legacy.go b/pkg/e2e/legacy.go index 47012b4f3b..3d75650f2f 100644 --- a/pkg/e2e/legacy.go +++ b/pkg/e2e/legacy.go @@ -81,7 +81,7 @@ func (crud *IngressCRUD) Delete(ns, name string) error { } func (crud *IngressCRUD) supportsNewAPI() (bool, error) { - if apiList, err := crud.C.Discovery().ServerResourcesForGroupVersion("networking/v1beta1"); err == nil { + if apiList, err := crud.C.Discovery().ServerResourcesForGroupVersion("networking.k8s.io/v1beta1"); err == nil { for _, r := range apiList.APIResources { if r.Kind == "Ingress" { return true, nil diff --git a/pkg/fuzz/helpers.go b/pkg/fuzz/helpers.go index 339d297bc8..455de3199d 100644 --- a/pkg/fuzz/helpers.go +++ b/pkg/fuzz/helpers.go @@ -258,6 +258,15 @@ func (i *IngressBuilder) AddStaticIP(name string) *IngressBuilder { return i } +// SetIngressClass sets Ingress class to given name. +func (i *IngressBuilder) SetIngressClass(name string) *IngressBuilder { + if i.ing.Annotations == nil { + i.ing.Annotations = make(map[string]string) + } + i.ing.Annotations[annotations.IngressClassKey] = name + return i +} + // BackendConfigBuilder is syntactic sugar for creating BackendConfig specs for testing // purposes. //