From 147e6089c1c56303f2be44ada920bae76f7e121c Mon Sep 17 00:00:00 2001 From: jm96441n Date: Mon, 5 Jun 2023 16:46:57 -0400 Subject: [PATCH 1/6] first pass at halting: got httproute and api-gateway done --- .../gateway_controller_integration_test.go | 556 ++++++++++++++++++ 1 file changed, 556 insertions(+) create mode 100644 control-plane/api-gateway/controllers/gateway_controller_integration_test.go diff --git a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go new file mode 100644 index 0000000000..ac7b5d3870 --- /dev/null +++ b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go @@ -0,0 +1,556 @@ +package controllers + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "testing" + "time" + + mapset "github.com/deckarep/golang-set" + logrtest "github.com/go-logr/logr/testr" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/hashicorp/consul-k8s/control-plane/api-gateway/cache" + "github.com/hashicorp/consul-k8s/control-plane/api-gateway/common" + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" + "github.com/hashicorp/consul-k8s/control-plane/helper/test" + "github.com/hashicorp/consul/api" +) + +func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { + s := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(s)) + require.NoError(t, gwv1alpha2.Install(s)) + require.NoError(t, gwv1beta1.Install(s)) + require.NoError(t, v1alpha1.AddToScheme(s)) + + k8sClient := registerFieldIndexersForTest(fake.NewClientBuilder().WithScheme(s)).Build() + consulTestServerClient := test.TestServerWithMockConnMgrWatcher(t, nil) + + ctx := context.Background() + logger := logrtest.New(t) + + cacheCfg := cache.Config{ + ConsulClientConfig: consulTestServerClient.Cfg, + ConsulServerConnMgr: consulTestServerClient.Watcher, + Logger: logger, + } + resourceCache := cache.New(cacheCfg) + + gwCache := cache.NewGatewayCache(ctx, cacheCfg) + + gwCtrl := GatewayController{ + HelmConfig: common.HelmConfig{}, + Log: logger, + Translator: common.ResourceTranslator{}, + cache: resourceCache, + gatewayCache: gwCache, + Client: k8sClient, + allowK8sNamespacesSet: mapset.NewSet(), + denyK8sNamespacesSet: mapset.NewSet(), + } + + go func() { + resourceCache.Run(ctx) + }() + + resourceCache.WaitSynced(ctx) + + gwSub := resourceCache.Subscribe(ctx, api.APIGateway, gwCtrl.transformConsulGateway) + httpRouteSub := resourceCache.Subscribe(ctx, api.HTTPRoute, gwCtrl.transformConsulHTTPRoute(ctx)) + _ = resourceCache.Subscribe(ctx, api.TCPRoute, gwCtrl.transformConsulTCPRoute(ctx)) + _ = resourceCache.Subscribe(ctx, api.InlineCertificate, gwCtrl.transformConsulInlineCertificate(ctx)) + + k8sGWObj := createAllFieldsSetAPIGW(t, ctx, k8sClient) + + // reconcile so we add the finalizer + _, err := gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + Name: k8sGWObj.Name, + }, + }) + require.NoError(t, err) + + // reconcile again so that we get the creation with the finalizer + _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + Name: k8sGWObj.Name, + }, + }) + require.NoError(t, err) + + httpRouteObj := createAllFieldsSetHTTPRoute(t, ctx, k8sClient, k8sGWObj) + // reconcile again so that we get the creation with the finalizer + _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + Name: k8sGWObj.Name, + }, + }) + + // reconcile again so that we get the creation with the finalizer + _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + Name: k8sGWObj.Name, + }, + }) + require.NoError(t, err) + + _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + Name: k8sGWObj.Name, + }, + }) + require.NoError(t, err) + + // get the creation events from the upsert + gwEvent := <-gwSub.Events() + <-httpRouteSub.Events() + + gwNamespaceName := types.NamespacedName{ + Name: gwEvent.Object.GetName(), + Namespace: gwEvent.Object.GetNamespace(), + } + + httpRouteNamespaceName := types.NamespacedName{ + Name: httpRouteObj.GetName(), + Namespace: httpRouteObj.GetNamespace(), + } + + gwRef := gwCtrl.Translator.ConfigEntryReference(api.APIGateway, gwNamespaceName) + httpRouteRef := gwCtrl.Translator.ConfigEntryReference(api.HTTPRoute, httpRouteNamespaceName) + curGWModifyIndex := resourceCache.Get(gwRef).GetModifyIndex() + curHTTPRouteModifyIndex := resourceCache.Get(httpRouteRef).GetModifyIndex() + + err = k8sClient.Get(ctx, gwNamespaceName, k8sGWObj) + require.NoError(t, err) + curGWResourceVersion := k8sGWObj.ResourceVersion + + err = k8sClient.Get(ctx, httpRouteNamespaceName, httpRouteObj) + require.NoError(t, err) + curHTTPRouteResourceVersion := httpRouteObj.ResourceVersion + + go func() { + // reconcile gateway again without any changes + for i := 0; i < 5; i++ { + _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + }, + }) + require.NoError(t, err) + } + }() + + require.Never(t, func() bool { + err = k8sClient.Get(ctx, gwNamespaceName, k8sGWObj) + require.NoError(t, err) + newGWResourceVersion := k8sGWObj.ResourceVersion + + err = k8sClient.Get(ctx, httpRouteNamespaceName, httpRouteObj) + require.NoError(t, err) + newHTTPRouteResourceVersion := httpRouteObj.ResourceVersion + + return curGWModifyIndex == resourceCache.Get(gwRef).GetModifyIndex() && + curGWResourceVersion == newGWResourceVersion && + curHTTPRouteModifyIndex == resourceCache.Get(httpRouteRef).GetModifyIndex() && + curHTTPRouteResourceVersion == newHTTPRouteResourceVersion + }, time.Duration(2*time.Second), time.Duration(500*time.Millisecond), fmt.Sprintf("curGWModifyIndex: %d, newIndx: %d", curGWModifyIndex, resourceCache.Get(gwRef).GetModifyIndex()), + ) +} + +func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client.WithWatch) *gwv1beta1.Gateway { + // listener one configuration + listenerOneName := "listener-one" + listenerOneHostname := "*.consul.io" + listenerOnePort := 3366 + listenerOneProtocol := "https" + + // listener one tls config + listenerOneCertName := "one-cert" + listenerOneCertK8sNamespace := "" + cert := generateTestCertificate(t, listenerOneCertK8sNamespace, listenerOneCertName) + + err := k8sClient.Create(ctx, &cert) + require.NoError(t, err) + + // listener two configuration + listenerTwoName := "listener-two" + listenerTwoHostname := "*.consul.io" + listenerTwoPort := 5432 + listenerTwoProtocol := "http" + + // Write gw to k8s + gwClassCfg := &v1alpha1.GatewayClassConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: "GatewayClassConfig", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-class-config", + }, + Spec: v1alpha1.GatewayClassConfigSpec{}, + } + gwClass := &gwv1beta1.GatewayClass{ + TypeMeta: metav1.TypeMeta{ + Kind: "GatewayClass", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gatewayclass", + }, + Spec: gwv1beta1.GatewayClassSpec{ + ControllerName: "hashicorp.com/consul-api-gateway-controller", + ParametersRef: &gwv1beta1.ParametersReference{ + Group: "consul.hashicorp.com", + Kind: "GatewayClassConfig", + Name: "gateway-class-config", + }, + Description: new(string), + }, + } + gw := &gwv1beta1.Gateway{ + TypeMeta: metav1.TypeMeta{ + Kind: "Gateway", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "consul", + Annotations: make(map[string]string), + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gwv1beta1.ObjectName(gwClass.Name), + Listeners: []gwv1beta1.Listener{ + { + Name: gwv1beta1.SectionName(listenerOneName), + Hostname: common.PointerTo(gwv1beta1.Hostname(listenerOneHostname)), + Port: gwv1beta1.PortNumber(listenerOnePort), + Protocol: gwv1beta1.ProtocolType(listenerOneProtocol), + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{ + { + Name: gwv1beta1.ObjectName(listenerOneCertName), + Namespace: common.PointerTo(gwv1beta1.Namespace(listenerOneCertK8sNamespace)), + }, + }, + }, + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + // TODO: do we support "selector" here? + From: common.PointerTo(gwv1beta1.FromNamespaces("All")), + }, + }, + }, + { + Name: gwv1beta1.SectionName(listenerTwoName), + Hostname: common.PointerTo(gwv1beta1.Hostname(listenerTwoHostname)), + Port: gwv1beta1.PortNumber(listenerTwoPort), + Protocol: gwv1beta1.ProtocolType(listenerTwoProtocol), + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + // TODO: do we support "selector" here? + From: common.PointerTo(gwv1beta1.FromNamespaces("Same")), + }, + }, + }, + }, + }, + } + + err = k8sClient.Create(ctx, gwClassCfg) + require.NoError(t, err) + + err = k8sClient.Create(ctx, gwClass) + require.NoError(t, err) + + err = k8sClient.Create(ctx, gw) + require.NoError(t, err) + + return gw +} + +func createAllFieldsSetHTTPRoute(t *testing.T, ctx context.Context, k8sClient client.WithWatch, gw *gwv1beta1.Gateway) *gwv1beta1.HTTPRoute { + svcDefault := &v1alpha1.ServiceDefaults{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceDefaults", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + }, + Spec: v1alpha1.ServiceDefaultsSpec{ + Protocol: "http", + }, + } + + svc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + Labels: map[string]string{"app": "Service"}, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "high", + Protocol: "TCP", + Port: 8080, + }, + }, + Selector: map[string]string{"app": "Service"}, + }, + } + + serviceAccount := &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + }, + } + + deployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + Labels: map[string]string{"app": "Service"}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: common.PointerTo(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "Service"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{}, + }, + }, + } + + err := k8sClient.Create(ctx, svcDefault) + require.NoError(t, err) + + err = k8sClient.Create(ctx, svc) + require.NoError(t, err) + + err = k8sClient.Create(ctx, serviceAccount) + require.NoError(t, err) + + err = k8sClient.Create(ctx, deployment) + require.NoError(t, err) + + refGrant := gwv1beta1.ReferenceGrant{ + TypeMeta: metav1.TypeMeta{ + Kind: "ReferenceGrant", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "grant", + Namespace: "consul", + }, + Spec: gwv1beta1.ReferenceGrantSpec{ + From: []gwv1beta1.ReferenceGrantFrom{ + { + Group: "gateway.networking.k8s.io", + Kind: "HTTPRoute", + }, + }, + To: []gwv1beta1.ReferenceGrantTo{ + { + Group: "gateway.networking.k8s.io", + Kind: "Gateway", + }, + }, + }, + } + + err = k8sClient.Create(ctx, &refGrant) + require.NoError(t, err) + + route := &gwv1beta1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: "HTTPRoute", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "http-route", + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Kind: (*gwv1beta1.Kind)(&gw.Kind), + Namespace: (*gwv1beta1.Namespace)(&gw.Namespace), + Name: gwv1beta1.ObjectName(gw.Name), + SectionName: &gw.Spec.Listeners[0].Name, + Port: &gw.Spec.Listeners[0].Port, + }, + }, + }, + Hostnames: []gwv1beta1.Hostname{"route.consul.io"}, + Rules: []gwv1beta1.HTTPRouteRule{ + { + Matches: []gwv1beta1.HTTPRouteMatch{ + { + Path: &gwv1beta1.HTTPPathMatch{ + Type: common.PointerTo(gwv1beta1.PathMatchType("PathPrefix")), + Value: common.PointerTo("/v1"), + }, + Headers: []gwv1beta1.HTTPHeaderMatch{ + { + Type: common.PointerTo(gwv1beta1.HeaderMatchExact), + Name: "version", + Value: "version", + }, + }, + QueryParams: []gwv1beta1.HTTPQueryParamMatch{ + { + Type: common.PointerTo(gwv1beta1.QueryParamMatchExact), + Name: "search", + Value: "q", + }, + }, + Method: common.PointerTo(gwv1beta1.HTTPMethod("GET")), + }, + }, + Filters: []gwv1beta1.HTTPRouteFilter{ + { + Type: gwv1beta1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gwv1beta1.HTTPHeaderFilter{ + Set: []gwv1beta1.HTTPHeader{ + { + Name: "foo", + Value: "bax", + }, + }, + Add: []gwv1beta1.HTTPHeader{ + { + Name: "arc", + Value: "reactor", + }, + }, + Remove: []string{"remove"}, + }, + }, + { + Type: gwv1beta1.HTTPRouteFilterURLRewrite, + URLRewrite: &gwv1beta1.HTTPURLRewriteFilter{ + Hostname: common.PointerTo(gwv1beta1.PreciseHostname("host.com")), + Path: &gwv1beta1.HTTPPathModifier{ + Type: gwv1beta1.FullPathHTTPPathModifier, + ReplaceFullPath: common.PointerTo("/foobar"), + }, + }, + }, + + { + Type: gwv1beta1.HTTPRouteFilterURLRewrite, + URLRewrite: &gwv1beta1.HTTPURLRewriteFilter{ + Hostname: common.PointerTo(gwv1beta1.PreciseHostname("host.com")), + Path: &gwv1beta1.HTTPPathModifier{ + Type: gwv1beta1.PrefixMatchHTTPPathModifier, + ReplacePrefixMatch: common.PointerTo("/foo"), + }, + }, + }, + }, + BackendRefs: []gwv1beta1.HTTPBackendRef{ + { + BackendRef: gwv1beta1.BackendRef{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: "Service", + Port: common.PointerTo(gwv1beta1.PortNumber(8080)), + }, + Weight: common.PointerTo(int32(50)), + }, + }, + }, + }, + }, + }, + } + + err = k8sClient.Create(ctx, route) + require.NoError(t, err) + + return route +} + +func createAllFieldsSetTCPRoute(t *testing.T, ctx context.Context, k8sClient client.WithWatch, gw *gwv1beta1.Gateway) *v1alpha2.TCPRoute { + return &v1alpha2.TCPRoute{} +} + +func generateTestCertificate(t *testing.T, namespace, name string) corev1.Secret { + privateKey, err := rsa.GenerateKey(rand.Reader, 1024) + require.NoError(t, err) + + usage := x509.KeyUsageCertSign + expiration := time.Now().AddDate(10, 0, 0) + + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "consul.test", + }, + IsCA: true, + NotBefore: time.Now().Add(-10 * time.Minute), + NotAfter: expiration, + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: usage, + BasicConstraintsValid: true, + } + caCert := cert + caPrivateKey := privateKey + + data, err := x509.CreateCertificate(rand.Reader, cert, caCert, &privateKey.PublicKey, caPrivateKey) + require.NoError(t, err) + + certBytes := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: data, + }) + + privateKeyBytes := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + }) + + return corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Data: map[string][]byte{ + corev1.TLSCertKey: certBytes, + corev1.TLSPrivateKeyKey: privateKeyBytes, + }, + } +} From 4e18f096ab89ab68f38f70a21cfeea402006560d Mon Sep 17 00:00:00 2001 From: jm96441n Date: Tue, 6 Jun 2023 10:44:42 -0400 Subject: [PATCH 2/6] clean up test --- .../gateway_controller_integration_test.go | 30 +++++-------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go index ac7b5d3870..175b83833c 100644 --- a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go +++ b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go @@ -80,6 +80,7 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { _ = resourceCache.Subscribe(ctx, api.InlineCertificate, gwCtrl.transformConsulInlineCertificate(ctx)) k8sGWObj := createAllFieldsSetAPIGW(t, ctx, k8sClient) + httpRouteObj := createAllFieldsSetHTTPRoute(t, ctx, k8sClient, k8sGWObj) // reconcile so we add the finalizer _, err := gwCtrl.Reconcile(ctx, reconcile.Request{ @@ -99,24 +100,7 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { }) require.NoError(t, err) - httpRouteObj := createAllFieldsSetHTTPRoute(t, ctx, k8sClient, k8sGWObj) - // reconcile again so that we get the creation with the finalizer - _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: k8sGWObj.Namespace, - Name: k8sGWObj.Name, - }, - }) - - // reconcile again so that we get the creation with the finalizer - _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: k8sGWObj.Namespace, - Name: k8sGWObj.Name, - }, - }) - require.NoError(t, err) - + // reconcile again so that we get the route bound to the gateway _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: k8sGWObj.Namespace, @@ -126,17 +110,17 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { require.NoError(t, err) // get the creation events from the upsert - gwEvent := <-gwSub.Events() + <-gwSub.Events() <-httpRouteSub.Events() gwNamespaceName := types.NamespacedName{ - Name: gwEvent.Object.GetName(), - Namespace: gwEvent.Object.GetNamespace(), + Name: k8sGWObj.Name, + Namespace: k8sGWObj.Namespace, } httpRouteNamespaceName := types.NamespacedName{ - Name: httpRouteObj.GetName(), - Namespace: httpRouteObj.GetNamespace(), + Name: httpRouteObj.Name, + Namespace: httpRouteObj.Namespace, } gwRef := gwCtrl.Translator.ConfigEntryReference(api.APIGateway, gwNamespaceName) From 45b8500287d5c22b7510808c0537dae10f76d26c Mon Sep 17 00:00:00 2001 From: jm96441n Date: Wed, 7 Jun 2023 15:17:05 -0400 Subject: [PATCH 3/6] Handle all set for infinite reconcile check --- .../gateway_controller_integration_test.go | 241 ++++++++++++++---- 1 file changed, 186 insertions(+), 55 deletions(-) diff --git a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go index 175b83833c..c7ea2a7c04 100644 --- a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go +++ b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go @@ -9,6 +9,7 @@ import ( "encoding/pem" "fmt" "math/big" + "sync" "testing" "time" @@ -45,7 +46,7 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { k8sClient := registerFieldIndexersForTest(fake.NewClientBuilder().WithScheme(s)).Build() consulTestServerClient := test.TestServerWithMockConnMgrWatcher(t, nil) - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) logger := logrtest.New(t) cacheCfg := cache.Config{ @@ -76,11 +77,10 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { gwSub := resourceCache.Subscribe(ctx, api.APIGateway, gwCtrl.transformConsulGateway) httpRouteSub := resourceCache.Subscribe(ctx, api.HTTPRoute, gwCtrl.transformConsulHTTPRoute(ctx)) - _ = resourceCache.Subscribe(ctx, api.TCPRoute, gwCtrl.transformConsulTCPRoute(ctx)) - _ = resourceCache.Subscribe(ctx, api.InlineCertificate, gwCtrl.transformConsulInlineCertificate(ctx)) + tcpRouteSub := resourceCache.Subscribe(ctx, api.TCPRoute, gwCtrl.transformConsulTCPRoute(ctx)) + inlineCertSub := resourceCache.Subscribe(ctx, api.InlineCertificate, gwCtrl.transformConsulInlineCertificate(ctx)) k8sGWObj := createAllFieldsSetAPIGW(t, ctx, k8sClient) - httpRouteObj := createAllFieldsSetHTTPRoute(t, ctx, k8sClient, k8sGWObj) // reconcile so we add the finalizer _, err := gwCtrl.Reconcile(ctx, reconcile.Request{ @@ -100,6 +100,18 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { }) require.NoError(t, err) + cert := createAllFieldsSetCert(t, ctx, k8sClient) + httpRouteObj := createAllFieldsSetHTTPRoute(t, ctx, k8sClient, k8sGWObj) + tcpRouteObj := createAllFieldsSetTCPRoute(t, ctx, k8sClient, k8sGWObj) + // reconcile again so that we get the route bound to the gateway + _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + Name: k8sGWObj.Name, + }, + }) + require.NoError(t, err) + // reconcile again so that we get the route bound to the gateway _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ NamespacedName: types.NamespacedName{ @@ -109,10 +121,44 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { }) require.NoError(t, err) - // get the creation events from the upsert - <-gwSub.Events() - <-httpRouteSub.Events() + wg := &sync.WaitGroup{} + wg.Add(3) + // wg.Add(4) + go func(w *sync.WaitGroup) { + gwDone := false + httpRouteDone := false + tcpRouteDone := false + inlineCertDone := false + for { + // get the creation events from the upsert and then continually read from channel so we dont block other subs + select { + case <-ctx.Done(): + return + case <-gwSub.Events(): + if !gwDone { + gwDone = true + wg.Done() + } + case <-httpRouteSub.Events(): + if !httpRouteDone { + httpRouteDone = true + wg.Done() + } + case <-tcpRouteSub.Events(): + if !tcpRouteDone { + tcpRouteDone = true + wg.Done() + } + case <-inlineCertSub.Events(): + if !inlineCertDone { + inlineCertDone = true + wg.Done() + } + } + } + }(wg) + wg.Wait() gwNamespaceName := types.NamespacedName{ Name: k8sGWObj.Name, Namespace: k8sGWObj.Namespace, @@ -123,10 +169,25 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { Namespace: httpRouteObj.Namespace, } + tcpRouteNamespaceName := types.NamespacedName{ + Name: tcpRouteObj.Name, + Namespace: tcpRouteObj.Namespace, + } + + certNamespaceName := types.NamespacedName{ + Name: cert.Name, + Namespace: cert.Namespace, + } + gwRef := gwCtrl.Translator.ConfigEntryReference(api.APIGateway, gwNamespaceName) httpRouteRef := gwCtrl.Translator.ConfigEntryReference(api.HTTPRoute, httpRouteNamespaceName) + tcpRouteRef := gwCtrl.Translator.ConfigEntryReference(api.TCPRoute, tcpRouteNamespaceName) + certRef := gwCtrl.Translator.ConfigEntryReference(api.InlineCertificate, certNamespaceName) + curGWModifyIndex := resourceCache.Get(gwRef).GetModifyIndex() curHTTPRouteModifyIndex := resourceCache.Get(httpRouteRef).GetModifyIndex() + curTCPRouteModifyIndex := resourceCache.Get(tcpRouteRef).GetModifyIndex() + curCertModifyIndex := resourceCache.Get(certRef).GetModifyIndex() err = k8sClient.Get(ctx, gwNamespaceName, k8sGWObj) require.NoError(t, err) @@ -136,8 +197,16 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { require.NoError(t, err) curHTTPRouteResourceVersion := httpRouteObj.ResourceVersion + err = k8sClient.Get(ctx, tcpRouteNamespaceName, tcpRouteObj) + require.NoError(t, err) + curTCPRouteResourceVersion := tcpRouteObj.ResourceVersion + + err = k8sClient.Get(ctx, certNamespaceName, cert) + require.NoError(t, err) + curCertResourceVersion := cert.ResourceVersion + go func() { - // reconcile gateway again without any changes + // reconcile multiple times with no changes to be sure for i := 0; i < 5; i++ { _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ NamespacedName: types.NamespacedName{ @@ -157,12 +226,25 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { require.NoError(t, err) newHTTPRouteResourceVersion := httpRouteObj.ResourceVersion + err = k8sClient.Get(ctx, tcpRouteNamespaceName, tcpRouteObj) + require.NoError(t, err) + newTCPRouteResourceVersion := tcpRouteObj.ResourceVersion + + err = k8sClient.Get(ctx, certNamespaceName, cert) + require.NoError(t, err) + newCertResourceVersion := cert.ResourceVersion + return curGWModifyIndex == resourceCache.Get(gwRef).GetModifyIndex() && curGWResourceVersion == newGWResourceVersion && curHTTPRouteModifyIndex == resourceCache.Get(httpRouteRef).GetModifyIndex() && - curHTTPRouteResourceVersion == newHTTPRouteResourceVersion + curHTTPRouteResourceVersion == newHTTPRouteResourceVersion && + curTCPRouteModifyIndex == resourceCache.Get(tcpRouteRef).GetModifyIndex() && + curTCPRouteResourceVersion == newTCPRouteResourceVersion && + curCertModifyIndex == resourceCache.Get(certRef).GetModifyIndex() && + curCertResourceVersion == newCertResourceVersion }, time.Duration(2*time.Second), time.Duration(500*time.Millisecond), fmt.Sprintf("curGWModifyIndex: %d, newIndx: %d", curGWModifyIndex, resourceCache.Get(gwRef).GetModifyIndex()), ) + cancel() } func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client.WithWatch) *gwv1beta1.Gateway { @@ -172,20 +254,23 @@ func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client listenerOnePort := 3366 listenerOneProtocol := "https" - // listener one tls config - listenerOneCertName := "one-cert" - listenerOneCertK8sNamespace := "" - cert := generateTestCertificate(t, listenerOneCertK8sNamespace, listenerOneCertName) - - err := k8sClient.Create(ctx, &cert) - require.NoError(t, err) - // listener two configuration listenerTwoName := "listener-two" listenerTwoHostname := "*.consul.io" listenerTwoPort := 5432 listenerTwoProtocol := "http" + // listener three configuration + listenerThreeName := "listener-three" + listenerThreePort := 8081 + listenerThreeProtocol := "tcp" + + // listener four configuration + listenerFourName := "listener-four" + listenerFourHostname := "*.consul.io" + listenerFourPort := 5433 + listenerFourProtocol := "http" + // Write gw to k8s gwClassCfg := &v1alpha1.GatewayClassConfig{ TypeMeta: metav1.TypeMeta{ @@ -236,14 +321,14 @@ func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client TLS: &gwv1beta1.GatewayTLSConfig{ CertificateRefs: []gwv1beta1.SecretObjectReference{ { - Name: gwv1beta1.ObjectName(listenerOneCertName), - Namespace: common.PointerTo(gwv1beta1.Namespace(listenerOneCertK8sNamespace)), + Kind: common.PointerTo(gwv1beta1.Kind("Secret")), + Name: gwv1beta1.ObjectName("one-cert"), + Namespace: common.PointerTo(gwv1beta1.Namespace("consul")), }, }, }, AllowedRoutes: &gwv1beta1.AllowedRoutes{ Namespaces: &gwv1beta1.RouteNamespaces{ - // TODO: do we support "selector" here? From: common.PointerTo(gwv1beta1.FromNamespaces("All")), }, }, @@ -255,16 +340,42 @@ func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client Protocol: gwv1beta1.ProtocolType(listenerTwoProtocol), AllowedRoutes: &gwv1beta1.AllowedRoutes{ Namespaces: &gwv1beta1.RouteNamespaces{ - // TODO: do we support "selector" here? From: common.PointerTo(gwv1beta1.FromNamespaces("Same")), }, }, }, + { + Name: gwv1beta1.SectionName(listenerThreeName), + Port: gwv1beta1.PortNumber(listenerThreePort), + Protocol: gwv1beta1.ProtocolType(listenerThreeProtocol), + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: common.PointerTo(gwv1beta1.FromNamespaces("All")), + }, + }, + }, + { + Name: gwv1beta1.SectionName(listenerFourName), + Hostname: common.PointerTo(gwv1beta1.Hostname(listenerFourHostname)), + Port: gwv1beta1.PortNumber(listenerFourPort), + Protocol: gwv1beta1.ProtocolType(listenerFourProtocol), + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: common.PointerTo(gwv1beta1.FromNamespaces("Selector")), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + common.NamespaceNameLabel: "consul", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{}, + }, + }, + }, + }, }, }, } - err = k8sClient.Create(ctx, gwClassCfg) + err := k8sClient.Create(ctx, gwClassCfg) require.NoError(t, err) err = k8sClient.Create(ctx, gwClass) @@ -350,34 +461,6 @@ func createAllFieldsSetHTTPRoute(t *testing.T, ctx context.Context, k8sClient cl err = k8sClient.Create(ctx, deployment) require.NoError(t, err) - refGrant := gwv1beta1.ReferenceGrant{ - TypeMeta: metav1.TypeMeta{ - Kind: "ReferenceGrant", - APIVersion: "gateway.networking.k8s.io/v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "grant", - Namespace: "consul", - }, - Spec: gwv1beta1.ReferenceGrantSpec{ - From: []gwv1beta1.ReferenceGrantFrom{ - { - Group: "gateway.networking.k8s.io", - Kind: "HTTPRoute", - }, - }, - To: []gwv1beta1.ReferenceGrantTo{ - { - Group: "gateway.networking.k8s.io", - Kind: "Gateway", - }, - }, - }, - } - - err = k8sClient.Create(ctx, &refGrant) - require.NoError(t, err) - route := &gwv1beta1.HTTPRoute{ TypeMeta: metav1.TypeMeta{ Kind: "HTTPRoute", @@ -488,10 +571,53 @@ func createAllFieldsSetHTTPRoute(t *testing.T, ctx context.Context, k8sClient cl } func createAllFieldsSetTCPRoute(t *testing.T, ctx context.Context, k8sClient client.WithWatch, gw *gwv1beta1.Gateway) *v1alpha2.TCPRoute { - return &v1alpha2.TCPRoute{} + route := &v1alpha2.TCPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: "TCPRoute", + APIVersion: "gateway.networking.k8s.io/v1alpha2", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-route", + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Kind: (*gwv1beta1.Kind)(&gw.Kind), + Namespace: (*gwv1beta1.Namespace)(&gw.Namespace), + Name: gwv1beta1.ObjectName(gw.Name), + SectionName: &gw.Spec.Listeners[2].Name, + Port: &gw.Spec.Listeners[2].Port, + }, + }, + }, + Rules: []gwv1alpha2.TCPRouteRule{ + { + BackendRefs: []gwv1beta1.BackendRef{ + { + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: "Service", + Port: common.PointerTo(gwv1beta1.PortNumber(25000)), + }, + Weight: common.PointerTo(int32(50)), + }, + }, + }, + }, + }, + } + + err := k8sClient.Create(ctx, route) + require.NoError(t, err) + + return route } -func generateTestCertificate(t *testing.T, namespace, name string) corev1.Secret { +func createAllFieldsSetCert(t *testing.T, ctx context.Context, k8sClient client.WithWatch) *corev1.Secret { + // listener one tls config + certName := "one-cert" + certNS := "consul" + privateKey, err := rsa.GenerateKey(rand.Reader, 1024) require.NoError(t, err) @@ -527,14 +653,19 @@ func generateTestCertificate(t *testing.T, namespace, name string) corev1.Secret Bytes: x509.MarshalPKCS1PrivateKey(privateKey), }) - return corev1.Secret{ + secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, + Namespace: certNS, + Name: certName, }, Data: map[string][]byte{ corev1.TLSCertKey: certBytes, corev1.TLSPrivateKeyKey: privateKeyBytes, }, } + + err = k8sClient.Create(ctx, secret) + require.NoError(t, err) + + return secret } From 8daf2aa90f8ea55dc406928d3ee8c2f9b9135349 Mon Sep 17 00:00:00 2001 From: jm96441n Date: Thu, 8 Jun 2023 16:33:41 -0400 Subject: [PATCH 4/6] Add table tests for minimal setup --- .../gateway_controller_integration_test.go | 646 +++++++++++++----- 1 file changed, 461 insertions(+), 185 deletions(-) diff --git a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go index c7ea2a7c04..a2e3bf7bea 100644 --- a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go +++ b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go @@ -43,211 +43,234 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { require.NoError(t, gwv1beta1.Install(s)) require.NoError(t, v1alpha1.AddToScheme(s)) - k8sClient := registerFieldIndexersForTest(fake.NewClientBuilder().WithScheme(s)).Build() - consulTestServerClient := test.TestServerWithMockConnMgrWatcher(t, nil) + testCases := map[string]struct { + namespace string + certFn func(*testing.T, context.Context, client.WithWatch, string) *corev1.Secret + gwFn func(*testing.T, context.Context, client.WithWatch, string) *gwv1beta1.Gateway + httpRouteFn func(*testing.T, context.Context, client.WithWatch, *gwv1beta1.Gateway) *gwv1beta1.HTTPRoute + tcpRouteFn func(*testing.T, context.Context, client.WithWatch, *gwv1beta1.Gateway) *v1alpha2.TCPRoute + }{ + "all fields set": { + namespace: "consul", + certFn: createCert, + gwFn: createAllFieldsSetAPIGW, + httpRouteFn: createAllFieldsSetHTTPRoute, + tcpRouteFn: createAllFieldsSetTCPRoute, + }, + "minimal fields set": { + namespace: "", + certFn: createCert, + gwFn: minimalFieldsSetAPIGW, + httpRouteFn: minimalFieldsSetHTTPRoute, + tcpRouteFn: minimalFieldsSetTCPRoute, + }, + } - ctx, cancel := context.WithCancel(context.Background()) - logger := logrtest.New(t) + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + k8sClient := registerFieldIndexersForTest(fake.NewClientBuilder().WithScheme(s)).Build() + consulTestServerClient := test.TestServerWithMockConnMgrWatcher(t, nil) + ctx, cancel := context.WithCancel(context.Background()) + logger := logrtest.New(t) + + cacheCfg := cache.Config{ + ConsulClientConfig: consulTestServerClient.Cfg, + ConsulServerConnMgr: consulTestServerClient.Watcher, + Logger: logger, + } + resourceCache := cache.New(cacheCfg) + + gwCache := cache.NewGatewayCache(ctx, cacheCfg) + + gwCtrl := GatewayController{ + HelmConfig: common.HelmConfig{}, + Log: logger, + Translator: common.ResourceTranslator{}, + cache: resourceCache, + gatewayCache: gwCache, + Client: k8sClient, + allowK8sNamespacesSet: mapset.NewSet(), + denyK8sNamespacesSet: mapset.NewSet(), + } - cacheCfg := cache.Config{ - ConsulClientConfig: consulTestServerClient.Cfg, - ConsulServerConnMgr: consulTestServerClient.Watcher, - Logger: logger, - } - resourceCache := cache.New(cacheCfg) - - gwCache := cache.NewGatewayCache(ctx, cacheCfg) - - gwCtrl := GatewayController{ - HelmConfig: common.HelmConfig{}, - Log: logger, - Translator: common.ResourceTranslator{}, - cache: resourceCache, - gatewayCache: gwCache, - Client: k8sClient, - allowK8sNamespacesSet: mapset.NewSet(), - denyK8sNamespacesSet: mapset.NewSet(), - } + go func() { + resourceCache.Run(ctx) + }() - go func() { - resourceCache.Run(ctx) - }() + resourceCache.WaitSynced(ctx) - resourceCache.WaitSynced(ctx) + gwSub := resourceCache.Subscribe(ctx, api.APIGateway, gwCtrl.transformConsulGateway) + httpRouteSub := resourceCache.Subscribe(ctx, api.HTTPRoute, gwCtrl.transformConsulHTTPRoute(ctx)) + tcpRouteSub := resourceCache.Subscribe(ctx, api.TCPRoute, gwCtrl.transformConsulTCPRoute(ctx)) + inlineCertSub := resourceCache.Subscribe(ctx, api.InlineCertificate, gwCtrl.transformConsulInlineCertificate(ctx)) - gwSub := resourceCache.Subscribe(ctx, api.APIGateway, gwCtrl.transformConsulGateway) - httpRouteSub := resourceCache.Subscribe(ctx, api.HTTPRoute, gwCtrl.transformConsulHTTPRoute(ctx)) - tcpRouteSub := resourceCache.Subscribe(ctx, api.TCPRoute, gwCtrl.transformConsulTCPRoute(ctx)) - inlineCertSub := resourceCache.Subscribe(ctx, api.InlineCertificate, gwCtrl.transformConsulInlineCertificate(ctx)) + cert := tc.certFn(t, ctx, k8sClient, tc.namespace) + k8sGWObj := tc.gwFn(t, ctx, k8sClient, tc.namespace) - k8sGWObj := createAllFieldsSetAPIGW(t, ctx, k8sClient) + // reconcile so we add the finalizer + _, err := gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + Name: k8sGWObj.Name, + }, + }) + require.NoError(t, err) - // reconcile so we add the finalizer - _, err := gwCtrl.Reconcile(ctx, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: k8sGWObj.Namespace, - Name: k8sGWObj.Name, - }, - }) - require.NoError(t, err) + // reconcile again so that we get the creation with the finalizer + _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + Name: k8sGWObj.Name, + }, + }) + require.NoError(t, err) - // reconcile again so that we get the creation with the finalizer - _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: k8sGWObj.Namespace, - Name: k8sGWObj.Name, - }, - }) - require.NoError(t, err) + httpRouteObj := tc.httpRouteFn(t, ctx, k8sClient, k8sGWObj) + tcpRouteObj := tc.tcpRouteFn(t, ctx, k8sClient, k8sGWObj) - cert := createAllFieldsSetCert(t, ctx, k8sClient) - httpRouteObj := createAllFieldsSetHTTPRoute(t, ctx, k8sClient, k8sGWObj) - tcpRouteObj := createAllFieldsSetTCPRoute(t, ctx, k8sClient, k8sGWObj) - // reconcile again so that we get the route bound to the gateway - _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: k8sGWObj.Namespace, - Name: k8sGWObj.Name, - }, - }) - require.NoError(t, err) + // reconcile again so that we get the route bound to the gateway + _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + Name: k8sGWObj.Name, + }, + }) + require.NoError(t, err) - // reconcile again so that we get the route bound to the gateway - _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: k8sGWObj.Namespace, - Name: k8sGWObj.Name, - }, - }) - require.NoError(t, err) + // reconcile again so that we get the route bound to the gateway + _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + Name: k8sGWObj.Name, + }, + }) + require.NoError(t, err) - wg := &sync.WaitGroup{} - wg.Add(3) - // wg.Add(4) - go func(w *sync.WaitGroup) { - gwDone := false - httpRouteDone := false - tcpRouteDone := false - inlineCertDone := false - for { - // get the creation events from the upsert and then continually read from channel so we dont block other subs - select { - case <-ctx.Done(): - return - case <-gwSub.Events(): - if !gwDone { - gwDone = true - wg.Done() - } - case <-httpRouteSub.Events(): - if !httpRouteDone { - httpRouteDone = true - wg.Done() - } - case <-tcpRouteSub.Events(): - if !tcpRouteDone { - tcpRouteDone = true - wg.Done() + wg := &sync.WaitGroup{} + // we never get the event from the cert because when it's created there are no gateways that reference it + wg.Add(3) + go func(w *sync.WaitGroup) { + gwDone := false + httpRouteDone := false + tcpRouteDone := false + for { + // get the creation events from the upsert and then continually read from channel so we dont block other subs + select { + case <-ctx.Done(): + return + case <-gwSub.Events(): + if !gwDone { + gwDone = true + wg.Done() + } + case <-httpRouteSub.Events(): + if !httpRouteDone { + httpRouteDone = true + wg.Done() + } + case <-tcpRouteSub.Events(): + if !tcpRouteDone { + tcpRouteDone = true + wg.Done() + } + case <-inlineCertSub.Events(): + } } - case <-inlineCertSub.Events(): - if !inlineCertDone { - inlineCertDone = true - wg.Done() - } - } - } - }(wg) + }(wg) - wg.Wait() - gwNamespaceName := types.NamespacedName{ - Name: k8sGWObj.Name, - Namespace: k8sGWObj.Namespace, - } + wg.Wait() - httpRouteNamespaceName := types.NamespacedName{ - Name: httpRouteObj.Name, - Namespace: httpRouteObj.Namespace, - } + gwNamespaceName := types.NamespacedName{ + Name: k8sGWObj.Name, + Namespace: k8sGWObj.Namespace, + } - tcpRouteNamespaceName := types.NamespacedName{ - Name: tcpRouteObj.Name, - Namespace: tcpRouteObj.Namespace, - } + httpRouteNamespaceName := types.NamespacedName{ + Name: httpRouteObj.Name, + Namespace: httpRouteObj.Namespace, + } - certNamespaceName := types.NamespacedName{ - Name: cert.Name, - Namespace: cert.Namespace, - } + tcpRouteNamespaceName := types.NamespacedName{ + Name: tcpRouteObj.Name, + Namespace: tcpRouteObj.Namespace, + } - gwRef := gwCtrl.Translator.ConfigEntryReference(api.APIGateway, gwNamespaceName) - httpRouteRef := gwCtrl.Translator.ConfigEntryReference(api.HTTPRoute, httpRouteNamespaceName) - tcpRouteRef := gwCtrl.Translator.ConfigEntryReference(api.TCPRoute, tcpRouteNamespaceName) - certRef := gwCtrl.Translator.ConfigEntryReference(api.InlineCertificate, certNamespaceName) + certNamespaceName := types.NamespacedName{ + Name: cert.Name, + Namespace: cert.Namespace, + } - curGWModifyIndex := resourceCache.Get(gwRef).GetModifyIndex() - curHTTPRouteModifyIndex := resourceCache.Get(httpRouteRef).GetModifyIndex() - curTCPRouteModifyIndex := resourceCache.Get(tcpRouteRef).GetModifyIndex() - curCertModifyIndex := resourceCache.Get(certRef).GetModifyIndex() + gwRef := gwCtrl.Translator.ConfigEntryReference(api.APIGateway, gwNamespaceName) + httpRouteRef := gwCtrl.Translator.ConfigEntryReference(api.HTTPRoute, httpRouteNamespaceName) + tcpRouteRef := gwCtrl.Translator.ConfigEntryReference(api.TCPRoute, tcpRouteNamespaceName) + certRef := gwCtrl.Translator.ConfigEntryReference(api.InlineCertificate, certNamespaceName) - err = k8sClient.Get(ctx, gwNamespaceName, k8sGWObj) - require.NoError(t, err) - curGWResourceVersion := k8sGWObj.ResourceVersion + curGWModifyIndex := resourceCache.Get(gwRef).GetModifyIndex() + curHTTPRouteModifyIndex := resourceCache.Get(httpRouteRef).GetModifyIndex() + curTCPRouteModifyIndex := resourceCache.Get(tcpRouteRef).GetModifyIndex() + curCertModifyIndex := resourceCache.Get(certRef).GetModifyIndex() - err = k8sClient.Get(ctx, httpRouteNamespaceName, httpRouteObj) - require.NoError(t, err) - curHTTPRouteResourceVersion := httpRouteObj.ResourceVersion + err = k8sClient.Get(ctx, gwNamespaceName, k8sGWObj) + require.NoError(t, err) + curGWResourceVersion := k8sGWObj.ResourceVersion - err = k8sClient.Get(ctx, tcpRouteNamespaceName, tcpRouteObj) - require.NoError(t, err) - curTCPRouteResourceVersion := tcpRouteObj.ResourceVersion + err = k8sClient.Get(ctx, httpRouteNamespaceName, httpRouteObj) + require.NoError(t, err) + curHTTPRouteResourceVersion := httpRouteObj.ResourceVersion - err = k8sClient.Get(ctx, certNamespaceName, cert) - require.NoError(t, err) - curCertResourceVersion := cert.ResourceVersion + err = k8sClient.Get(ctx, tcpRouteNamespaceName, tcpRouteObj) + require.NoError(t, err) + curTCPRouteResourceVersion := tcpRouteObj.ResourceVersion - go func() { - // reconcile multiple times with no changes to be sure - for i := 0; i < 5; i++ { - _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: k8sGWObj.Namespace, - }, - }) + err = k8sClient.Get(ctx, certNamespaceName, cert) require.NoError(t, err) - } - }() - - require.Never(t, func() bool { - err = k8sClient.Get(ctx, gwNamespaceName, k8sGWObj) - require.NoError(t, err) - newGWResourceVersion := k8sGWObj.ResourceVersion - - err = k8sClient.Get(ctx, httpRouteNamespaceName, httpRouteObj) - require.NoError(t, err) - newHTTPRouteResourceVersion := httpRouteObj.ResourceVersion - - err = k8sClient.Get(ctx, tcpRouteNamespaceName, tcpRouteObj) - require.NoError(t, err) - newTCPRouteResourceVersion := tcpRouteObj.ResourceVersion - - err = k8sClient.Get(ctx, certNamespaceName, cert) - require.NoError(t, err) - newCertResourceVersion := cert.ResourceVersion - - return curGWModifyIndex == resourceCache.Get(gwRef).GetModifyIndex() && - curGWResourceVersion == newGWResourceVersion && - curHTTPRouteModifyIndex == resourceCache.Get(httpRouteRef).GetModifyIndex() && - curHTTPRouteResourceVersion == newHTTPRouteResourceVersion && - curTCPRouteModifyIndex == resourceCache.Get(tcpRouteRef).GetModifyIndex() && - curTCPRouteResourceVersion == newTCPRouteResourceVersion && - curCertModifyIndex == resourceCache.Get(certRef).GetModifyIndex() && - curCertResourceVersion == newCertResourceVersion - }, time.Duration(2*time.Second), time.Duration(500*time.Millisecond), fmt.Sprintf("curGWModifyIndex: %d, newIndx: %d", curGWModifyIndex, resourceCache.Get(gwRef).GetModifyIndex()), - ) - cancel() + curCertResourceVersion := cert.ResourceVersion + + go func() { + // reconcile multiple times with no changes to be sure + for i := 0; i < 5; i++ { + _, err = gwCtrl.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: k8sGWObj.Namespace, + }, + }) + require.NoError(t, err) + } + }() + + require.Never(t, func() bool { + err = k8sClient.Get(ctx, gwNamespaceName, k8sGWObj) + require.NoError(t, err) + newGWResourceVersion := k8sGWObj.ResourceVersion + + err = k8sClient.Get(ctx, httpRouteNamespaceName, httpRouteObj) + require.NoError(t, err) + newHTTPRouteResourceVersion := httpRouteObj.ResourceVersion + + err = k8sClient.Get(ctx, tcpRouteNamespaceName, tcpRouteObj) + require.NoError(t, err) + newTCPRouteResourceVersion := tcpRouteObj.ResourceVersion + + err = k8sClient.Get(ctx, certNamespaceName, cert) + require.NoError(t, err) + newCertResourceVersion := cert.ResourceVersion + + return curGWModifyIndex == resourceCache.Get(gwRef).GetModifyIndex() && + curGWResourceVersion == newGWResourceVersion && + curHTTPRouteModifyIndex == resourceCache.Get(httpRouteRef).GetModifyIndex() && + curHTTPRouteResourceVersion == newHTTPRouteResourceVersion && + curTCPRouteModifyIndex == resourceCache.Get(tcpRouteRef).GetModifyIndex() && + curTCPRouteResourceVersion == newTCPRouteResourceVersion && + curCertModifyIndex == resourceCache.Get(certRef).GetModifyIndex() && + curCertResourceVersion == newCertResourceVersion + }, time.Duration(2*time.Second), time.Duration(500*time.Millisecond), fmt.Sprintf("curGWModifyIndex: %d, newIndx: %d", curGWModifyIndex, resourceCache.Get(gwRef).GetModifyIndex()), + ) + cancel() + }) + } } -func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client.WithWatch) *gwv1beta1.Gateway { +func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client.WithWatch, namespace string) *gwv1beta1.Gateway { // listener one configuration listenerOneName := "listener-one" listenerOneHostname := "*.consul.io" @@ -258,7 +281,7 @@ func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client listenerTwoName := "listener-two" listenerTwoHostname := "*.consul.io" listenerTwoPort := 5432 - listenerTwoProtocol := "http" + listenerTwoProtocol := "HTTP" // listener three configuration listenerThreeName := "listener-three" @@ -269,7 +292,7 @@ func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client listenerFourName := "listener-four" listenerFourHostname := "*.consul.io" listenerFourPort := 5433 - listenerFourProtocol := "http" + listenerFourProtocol := "HtTp" // Write gw to k8s gwClassCfg := &v1alpha1.GatewayClassConfig{ @@ -307,7 +330,7 @@ func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client }, ObjectMeta: metav1.ObjectMeta{ Name: "gw", - Namespace: "consul", + Namespace: namespace, Annotations: make(map[string]string), }, Spec: gwv1beta1.GatewaySpec{ @@ -323,7 +346,7 @@ func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client { Kind: common.PointerTo(gwv1beta1.Kind("Secret")), Name: gwv1beta1.ObjectName("one-cert"), - Namespace: common.PointerTo(gwv1beta1.Namespace("consul")), + Namespace: common.PointerTo(gwv1beta1.Namespace(namespace)), }, }, }, @@ -613,10 +636,9 @@ func createAllFieldsSetTCPRoute(t *testing.T, ctx context.Context, k8sClient cli return route } -func createAllFieldsSetCert(t *testing.T, ctx context.Context, k8sClient client.WithWatch) *corev1.Secret { +func createCert(t *testing.T, ctx context.Context, k8sClient client.WithWatch, certNS string) *corev1.Secret { // listener one tls config certName := "one-cert" - certNS := "consul" privateKey, err := rsa.GenerateKey(rand.Reader, 1024) require.NoError(t, err) @@ -669,3 +691,257 @@ func createAllFieldsSetCert(t *testing.T, ctx context.Context, k8sClient client. return secret } + +func minimalFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client.WithWatch, namespace string) *gwv1beta1.Gateway { + // listener one configuration + listenerOneName := "listener-one" + listenerOneHostname := "*.consul.io" + listenerOnePort := 3366 + listenerOneProtocol := "https" + + // listener three configuration + listenerThreeName := "listener-three" + listenerThreePort := 8081 + listenerThreeProtocol := "tcp" + + // Write gw to k8s + gwClassCfg := &v1alpha1.GatewayClassConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: "GatewayClassConfig", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-class-config", + }, + Spec: v1alpha1.GatewayClassConfigSpec{}, + } + gwClass := &gwv1beta1.GatewayClass{ + TypeMeta: metav1.TypeMeta{ + Kind: "GatewayClass", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gatewayclass", + }, + Spec: gwv1beta1.GatewayClassSpec{ + ControllerName: "hashicorp.com/consul-api-gateway-controller", + ParametersRef: &gwv1beta1.ParametersReference{ + Group: "consul.hashicorp.com", + Kind: "GatewayClassConfig", + Name: "gateway-class-config", + }, + Description: new(string), + }, + } + gw := &gwv1beta1.Gateway{ + TypeMeta: metav1.TypeMeta{ + Kind: "Gateway", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Annotations: make(map[string]string), + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gwv1beta1.ObjectName(gwClass.Name), + Listeners: []gwv1beta1.Listener{ + { + Name: gwv1beta1.SectionName(listenerOneName), + Hostname: common.PointerTo(gwv1beta1.Hostname(listenerOneHostname)), + Port: gwv1beta1.PortNumber(listenerOnePort), + Protocol: gwv1beta1.ProtocolType(listenerOneProtocol), + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{ + { + Kind: common.PointerTo(gwv1beta1.Kind("Secret")), + Name: gwv1beta1.ObjectName("one-cert"), + Namespace: common.PointerTo(gwv1beta1.Namespace(namespace)), + }, + }, + }, + }, + { + Name: gwv1beta1.SectionName(listenerThreeName), + Port: gwv1beta1.PortNumber(listenerThreePort), + Protocol: gwv1beta1.ProtocolType(listenerThreeProtocol), + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: common.PointerTo(gwv1beta1.FromNamespaces("All")), + }, + }, + }, + }, + }, + } + + err := k8sClient.Create(ctx, gwClassCfg) + require.NoError(t, err) + + err = k8sClient.Create(ctx, gwClass) + require.NoError(t, err) + + err = k8sClient.Create(ctx, gw) + require.NoError(t, err) + + return gw +} + +func minimalFieldsSetHTTPRoute(t *testing.T, ctx context.Context, k8sClient client.WithWatch, gw *gwv1beta1.Gateway) *gwv1beta1.HTTPRoute { + svcDefault := &v1alpha1.ServiceDefaults{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceDefaults", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + }, + Spec: v1alpha1.ServiceDefaultsSpec{ + Protocol: "http", + }, + } + + svc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + Labels: map[string]string{"app": "Service"}, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "high", + Protocol: "TCP", + Port: 8080, + }, + }, + Selector: map[string]string{"app": "Service"}, + }, + } + + serviceAccount := &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + }, + } + + deployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + Labels: map[string]string{"app": "Service"}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: common.PointerTo(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "Service"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{}, + }, + }, + } + + err := k8sClient.Create(ctx, svcDefault) + require.NoError(t, err) + + err = k8sClient.Create(ctx, svc) + require.NoError(t, err) + + err = k8sClient.Create(ctx, serviceAccount) + require.NoError(t, err) + + err = k8sClient.Create(ctx, deployment) + require.NoError(t, err) + + route := &gwv1beta1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: "HTTPRoute", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "http-route", + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Kind: (*gwv1beta1.Kind)(&gw.Kind), + Namespace: (*gwv1beta1.Namespace)(&gw.Namespace), + Name: gwv1beta1.ObjectName(gw.Name), + SectionName: &gw.Spec.Listeners[0].Name, + Port: &gw.Spec.Listeners[0].Port, + }, + }, + }, + Hostnames: []gwv1beta1.Hostname{"route.consul.io"}, + Rules: []gwv1beta1.HTTPRouteRule{ + { + BackendRefs: []gwv1beta1.HTTPBackendRef{ + { + BackendRef: gwv1beta1.BackendRef{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: "Service", + Port: common.PointerTo(gwv1beta1.PortNumber(8080)), + }, + }, + }, + }, + }, + }, + }, + } + + err = k8sClient.Create(ctx, route) + require.NoError(t, err) + + return route +} + +func minimalFieldsSetTCPRoute(t *testing.T, ctx context.Context, k8sClient client.WithWatch, gw *gwv1beta1.Gateway) *v1alpha2.TCPRoute { + route := &v1alpha2.TCPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: "TCPRoute", + APIVersion: "gateway.networking.k8s.io/v1alpha2", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-route", + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Kind: (*gwv1beta1.Kind)(&gw.Kind), + Namespace: (*gwv1beta1.Namespace)(&gw.Namespace), + Name: gwv1beta1.ObjectName(gw.Name), + SectionName: &gw.Spec.Listeners[1].Name, + Port: &gw.Spec.Listeners[1].Port, + }, + }, + }, + Rules: []gwv1alpha2.TCPRouteRule{ + { + BackendRefs: []gwv1beta1.BackendRef{ + { + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: "Service", + Port: common.PointerTo(gwv1beta1.PortNumber(25000)), + }, + }, + }, + }, + }, + }, + } + + err := k8sClient.Create(ctx, route) + require.NoError(t, err) + + return route +} From 04f1e691378ad12adc6b626926ffe6253d45a57f Mon Sep 17 00:00:00 2001 From: jm96441n Date: Fri, 9 Jun 2023 10:59:28 -0400 Subject: [PATCH 5/6] Added some odd field names to test normalization is handled correctly --- .../gateway_controller_integration_test.go | 374 +++++++++++++++++- 1 file changed, 372 insertions(+), 2 deletions(-) diff --git a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go index a2e3bf7bea..6d7d6cdafc 100644 --- a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go +++ b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go @@ -64,6 +64,13 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { httpRouteFn: minimalFieldsSetHTTPRoute, tcpRouteFn: minimalFieldsSetTCPRoute, }, + "funky casing to test normalization doesnt cause infinite reconciliation": { + namespace: "", + certFn: createCert, + gwFn: createFunkyCasingFieldsAPIGW, + httpRouteFn: createAllFieldsSetHTTPRoute, + tcpRouteFn: createFunkyCasingFieldsTCPRoute, + }, } for name, tc := range testCases { @@ -281,7 +288,7 @@ func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client listenerTwoName := "listener-two" listenerTwoHostname := "*.consul.io" listenerTwoPort := 5432 - listenerTwoProtocol := "HTTP" + listenerTwoProtocol := "http" // listener three configuration listenerThreeName := "listener-three" @@ -292,7 +299,7 @@ func createAllFieldsSetAPIGW(t *testing.T, ctx context.Context, k8sClient client listenerFourName := "listener-four" listenerFourHostname := "*.consul.io" listenerFourPort := 5433 - listenerFourProtocol := "HtTp" + listenerFourProtocol := "http" // Write gw to k8s gwClassCfg := &v1alpha1.GatewayClassConfig{ @@ -945,3 +952,366 @@ func minimalFieldsSetTCPRoute(t *testing.T, ctx context.Context, k8sClient clien return route } + +func createFunkyCasingFieldsAPIGW(t *testing.T, ctx context.Context, k8sClient client.WithWatch, namespace string) *gwv1beta1.Gateway { + // listener one configuration + listenerOneName := "listener-one" + listenerOneHostname := "*.consul.io" + listenerOnePort := 3366 + listenerOneProtocol := "hTtPs" + + // listener two configuration + listenerTwoName := "listener-two" + listenerTwoHostname := "*.consul.io" + listenerTwoPort := 5432 + listenerTwoProtocol := "HTTP" + + // listener three configuration + listenerThreeName := "listener-three" + listenerThreePort := 8081 + listenerThreeProtocol := "tCp" + + // listener four configuration + listenerFourName := "listener-four" + listenerFourHostname := "*.consul.io" + listenerFourPort := 5433 + listenerFourProtocol := "hTTp" + + // Write gw to k8s + gwClassCfg := &v1alpha1.GatewayClassConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: "GatewayClassConfig", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-class-config", + }, + Spec: v1alpha1.GatewayClassConfigSpec{}, + } + gwClass := &gwv1beta1.GatewayClass{ + TypeMeta: metav1.TypeMeta{ + Kind: "GatewayClass", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gatewayclass", + }, + Spec: gwv1beta1.GatewayClassSpec{ + ControllerName: "hashicorp.com/consul-api-gateway-controller", + ParametersRef: &gwv1beta1.ParametersReference{ + Group: "consul.hashicorp.com", + Kind: "GatewayClassConfig", + Name: "gateway-class-config", + }, + Description: new(string), + }, + } + gw := &gwv1beta1.Gateway{ + TypeMeta: metav1.TypeMeta{ + Kind: "Gateway", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: namespace, + Annotations: make(map[string]string), + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gwv1beta1.ObjectName(gwClass.Name), + Listeners: []gwv1beta1.Listener{ + { + Name: gwv1beta1.SectionName(listenerOneName), + Hostname: common.PointerTo(gwv1beta1.Hostname(listenerOneHostname)), + Port: gwv1beta1.PortNumber(listenerOnePort), + Protocol: gwv1beta1.ProtocolType(listenerOneProtocol), + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{ + { + Kind: common.PointerTo(gwv1beta1.Kind("Secret")), + Name: gwv1beta1.ObjectName("one-cert"), + Namespace: common.PointerTo(gwv1beta1.Namespace(namespace)), + }, + }, + }, + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: common.PointerTo(gwv1beta1.FromNamespaces("All")), + }, + }, + }, + { + Name: gwv1beta1.SectionName(listenerTwoName), + Hostname: common.PointerTo(gwv1beta1.Hostname(listenerTwoHostname)), + Port: gwv1beta1.PortNumber(listenerTwoPort), + Protocol: gwv1beta1.ProtocolType(listenerTwoProtocol), + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: common.PointerTo(gwv1beta1.FromNamespaces("Same")), + }, + }, + }, + { + Name: gwv1beta1.SectionName(listenerThreeName), + Port: gwv1beta1.PortNumber(listenerThreePort), + Protocol: gwv1beta1.ProtocolType(listenerThreeProtocol), + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: common.PointerTo(gwv1beta1.FromNamespaces("All")), + }, + }, + }, + { + Name: gwv1beta1.SectionName(listenerFourName), + Hostname: common.PointerTo(gwv1beta1.Hostname(listenerFourHostname)), + Port: gwv1beta1.PortNumber(listenerFourPort), + Protocol: gwv1beta1.ProtocolType(listenerFourProtocol), + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: common.PointerTo(gwv1beta1.FromNamespaces("Selector")), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + common.NamespaceNameLabel: "consul", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{}, + }, + }, + }, + }, + }, + }, + } + + err := k8sClient.Create(ctx, gwClassCfg) + require.NoError(t, err) + + err = k8sClient.Create(ctx, gwClass) + require.NoError(t, err) + + err = k8sClient.Create(ctx, gw) + require.NoError(t, err) + + return gw +} + +func createFunkyCasingFieldsHTTPRoute(t *testing.T, ctx context.Context, k8sClient client.WithWatch, gw *gwv1beta1.Gateway) *gwv1beta1.HTTPRoute { + svcDefault := &v1alpha1.ServiceDefaults{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceDefaults", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + }, + Spec: v1alpha1.ServiceDefaultsSpec{ + Protocol: "hTtp", + }, + } + + svc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + Labels: map[string]string{"app": "Service"}, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "high", + Protocol: "TCP", + Port: 8080, + }, + }, + Selector: map[string]string{"app": "Service"}, + }, + } + + serviceAccount := &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + }, + } + + deployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "Service", + Labels: map[string]string{"app": "Service"}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: common.PointerTo(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "Service"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{}, + }, + }, + } + + err := k8sClient.Create(ctx, svcDefault) + require.NoError(t, err) + + err = k8sClient.Create(ctx, svc) + require.NoError(t, err) + + err = k8sClient.Create(ctx, serviceAccount) + require.NoError(t, err) + + err = k8sClient.Create(ctx, deployment) + require.NoError(t, err) + + route := &gwv1beta1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: "HTTPRoute", + APIVersion: "gateway.networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "http-route", + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Namespace: (*gwv1beta1.Namespace)(&gw.Namespace), + Name: gwv1beta1.ObjectName(gw.Name), + SectionName: &gw.Spec.Listeners[0].Name, + Port: &gw.Spec.Listeners[0].Port, + }, + }, + }, + Hostnames: []gwv1beta1.Hostname{"route.consul.io"}, + Rules: []gwv1beta1.HTTPRouteRule{ + { + Matches: []gwv1beta1.HTTPRouteMatch{ + { + Path: &gwv1beta1.HTTPPathMatch{ + Type: common.PointerTo(gwv1beta1.PathMatchPathPrefix), + }, + Headers: []gwv1beta1.HTTPHeaderMatch{ + { + Type: common.PointerTo(gwv1beta1.HeaderMatchExact), + Name: "version", + Value: "version", + }, + }, + QueryParams: []gwv1beta1.HTTPQueryParamMatch{ + { + Type: common.PointerTo(gwv1beta1.QueryParamMatchExact), + Name: "search", + Value: "q", + }, + }, + Method: common.PointerTo(gwv1beta1.HTTPMethod("GET")), + }, + }, + Filters: []gwv1beta1.HTTPRouteFilter{ + { + Type: gwv1beta1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gwv1beta1.HTTPHeaderFilter{ + Set: []gwv1beta1.HTTPHeader{ + { + Name: "foo", + Value: "bax", + }, + }, + Add: []gwv1beta1.HTTPHeader{ + { + Name: "arc", + Value: "reactor", + }, + }, + Remove: []string{"remove"}, + }, + }, + { + Type: gwv1beta1.HTTPRouteFilterURLRewrite, + URLRewrite: &gwv1beta1.HTTPURLRewriteFilter{ + Hostname: common.PointerTo(gwv1beta1.PreciseHostname("host.com")), + Path: &gwv1beta1.HTTPPathModifier{ + Type: gwv1beta1.FullPathHTTPPathModifier, + ReplaceFullPath: common.PointerTo("/foobar"), + }, + }, + }, + + { + Type: gwv1beta1.HTTPRouteFilterURLRewrite, + URLRewrite: &gwv1beta1.HTTPURLRewriteFilter{ + Hostname: common.PointerTo(gwv1beta1.PreciseHostname("host.com")), + Path: &gwv1beta1.HTTPPathModifier{ + Type: gwv1beta1.PrefixMatchHTTPPathModifier, + ReplacePrefixMatch: common.PointerTo("/foo"), + }, + }, + }, + }, + BackendRefs: []gwv1beta1.HTTPBackendRef{ + { + BackendRef: gwv1beta1.BackendRef{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: "Service", + Port: common.PointerTo(gwv1beta1.PortNumber(8080)), + }, + Weight: common.PointerTo(int32(-50)), + }, + }, + }, + }, + }, + }, + } + + err = k8sClient.Create(ctx, route) + require.NoError(t, err) + + return route +} + +func createFunkyCasingFieldsTCPRoute(t *testing.T, ctx context.Context, k8sClient client.WithWatch, gw *gwv1beta1.Gateway) *v1alpha2.TCPRoute { + route := &v1alpha2.TCPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: "TCPRoute", + APIVersion: "gateway.networking.k8s.io/v1alpha2", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-route", + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Namespace: (*gwv1beta1.Namespace)(&gw.Namespace), + Name: gwv1beta1.ObjectName(gw.Name), + SectionName: &gw.Spec.Listeners[2].Name, + Port: &gw.Spec.Listeners[2].Port, + }, + }, + }, + Rules: []gwv1alpha2.TCPRouteRule{ + { + BackendRefs: []gwv1beta1.BackendRef{ + { + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: "Service", + Port: common.PointerTo(gwv1beta1.PortNumber(25000)), + }, + Weight: common.PointerTo(int32(-50)), + }, + }, + }, + }, + }, + } + + err := k8sClient.Create(ctx, route) + require.NoError(t, err) + + return route +} From 75eeb9f9aea86e0f4aef7a46f0e1154b23750f86 Mon Sep 17 00:00:00 2001 From: jm96441n Date: Fri, 9 Jun 2023 11:42:02 -0400 Subject: [PATCH 6/6] Use funky casing http routes --- .../controllers/gateway_controller_integration_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go index 6d7d6cdafc..5dd5357d77 100644 --- a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go +++ b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go @@ -68,7 +68,7 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { namespace: "", certFn: createCert, gwFn: createFunkyCasingFieldsAPIGW, - httpRouteFn: createAllFieldsSetHTTPRoute, + httpRouteFn: createFunkyCasingFieldsHTTPRoute, tcpRouteFn: createFunkyCasingFieldsTCPRoute, }, } @@ -78,6 +78,10 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { k8sClient := registerFieldIndexersForTest(fake.NewClientBuilder().WithScheme(s)).Build() consulTestServerClient := test.TestServerWithMockConnMgrWatcher(t, nil) ctx, cancel := context.WithCancel(context.Background()) + + t.Cleanup(func() { + cancel() + }) logger := logrtest.New(t) cacheCfg := cache.Config{ @@ -272,7 +276,6 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { curCertResourceVersion == newCertResourceVersion }, time.Duration(2*time.Second), time.Duration(500*time.Millisecond), fmt.Sprintf("curGWModifyIndex: %d, newIndx: %d", curGWModifyIndex, resourceCache.Get(gwRef).GetModifyIndex()), ) - cancel() }) } } @@ -1208,7 +1211,7 @@ func createFunkyCasingFieldsHTTPRoute(t *testing.T, ctx context.Context, k8sClie Value: "q", }, }, - Method: common.PointerTo(gwv1beta1.HTTPMethod("GET")), + Method: common.PointerTo(gwv1beta1.HTTPMethod("geT")), }, }, Filters: []gwv1beta1.HTTPRouteFilter{