diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index d07d1e993..00f573934 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -1,7 +1,6 @@ package main import ( - "errors" "fmt" "os" @@ -12,6 +11,7 @@ import ( "github.com/nginxinc/nginx-kubernetes-gateway/internal/config" "github.com/nginxinc/nginx-kubernetes-gateway/internal/manager" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/provisioner" ) const ( @@ -190,7 +190,10 @@ func createProvisionerModeCommand() *cobra.Command { "date", date, ) - return errors.New("not implemented yet") + return provisioner.StartManager(provisioner.Config{ + Logger: logger, + GatewayClassName: gatewayClassName.value, + }) }, } } diff --git a/conformance/provisioner/provisioner.md b/conformance/provisioner/provisioner.md new file mode 100644 index 000000000..9dadd31eb --- /dev/null +++ b/conformance/provisioner/provisioner.md @@ -0,0 +1,35 @@ +# Provisioner + +Provisioner implements data plane provisioning for NGINX Kubernetes Gateway (NKG): it creates an NKG static mode +Deployment for each Gateway that belongs to the provisioner GatewayClass. + +``` +Usage: + gateway provisioner-mode [flags] + +Flags: + -h, --help help for provisioner-mode + +Global Flags: + --gateway-ctlr-name string The name of the Gateway controller. The controller name must be of the form: DOMAIN/PATH. The controller's domain is 'k8s-gateway.nginx.org' (default "") + --gatewayclass string The name of the GatewayClass resource. Every NGINX Gateway must have a unique corresponding GatewayClass resource. (default "") +``` + +Provisioner is not meant to be used in production yet (see this issue for more details +https://github.com/nginxinc/nginx-kubernetes-gateway/issues/634). However, it can be used in the Gateway API conformance +tests, which expect a Gateway API implementation to provision an independent data plane per Gateway. + +How to deploy: + +1. Follow the [installation](/docs/installation.md) instructions up until the Deploy the NGINX Kubernetes Gateway Step + to deploy prerequisites for both the static mode Deployments and the provisioner. +1. Deploy provisioner: + ``` + kubectl apply -f conformance/provisioner/provisioner.yaml + ``` +1. Confirm the provisioner is running in nginx-gateway namespace: + ``` + kubectl get pods -n nginx-gateway + NAME READY STATUS RESTARTS AGE + nginx-gateway-provisioner-6c9d9fdcb8-b2pf8 1/1 Running 0 11m + ``` diff --git a/conformance/provisioner/provisioner.yaml b/conformance/provisioner/provisioner.yaml new file mode 100644 index 000000000..ddb4e53d0 --- /dev/null +++ b/conformance/provisioner/provisioner.yaml @@ -0,0 +1,72 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nginx-gateway-provisioner + namespace: nginx-gateway +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nginx-gateway-provisioner +rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + verbs: + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/status + verbs: + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nginx-gateway-provisioner +subjects: +- kind: ServiceAccount + name: nginx-gateway-provisioner + namespace: nginx-gateway +roleRef: + kind: ClusterRole + name: nginx-gateway-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-gateway-provisioner + namespace: nginx-gateway +spec: + replicas: 1 + selector: + matchLabels: + app: nginx-gateway-provisioner + template: + metadata: + labels: + app: nginx-gateway-provisioner + spec: + serviceAccountName: nginx-gateway-provisioner + containers: + - image: ghcr.io/nginxinc/nginx-kubernetes-gateway:edge + imagePullPolicy: Always + name: nginx-gateway-provisioner + securityContext: + runAsUser: 1001 + args: + - provisioner-mode + - --gateway-ctlr-name=k8s-gateway.nginx.org/nginx-gateway-controller + - --gatewayclass=nginx diff --git a/deploy/manifests/nginx-gateway.yaml b/deploy/manifests/deployment.yaml similarity index 59% rename from deploy/manifests/nginx-gateway.yaml rename to deploy/manifests/deployment.yaml index 13f0729c1..76816f32e 100644 --- a/deploy/manifests/nginx-gateway.yaml +++ b/deploy/manifests/deployment.yaml @@ -1,95 +1,3 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-gateway - namespace: nginx-gateway ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nginx-gateway -rules: -- apiGroups: - - "" - resources: - - services - - secrets - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses - - gateways - - httproutes - verbs: - - list - - watch -- apiGroups: - - gateway.nginx.org - resources: - - gatewayconfigs - verbs: - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - httproutes/status - - gateways/status - - gatewayclasses/status - verbs: - - update ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nginx-gateway -subjects: -- kind: ServiceAccount - name: nginx-gateway - namespace: nginx-gateway -roleRef: - kind: ClusterRole - name: nginx-gateway - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx-conf - namespace: nginx-gateway -data: - nginx.conf: | - load_module /usr/lib/nginx/modules/ngx_http_js_module.so; - - events {} - - pid /etc/nginx/nginx.pid; - error_log stderr debug; - - http { - include /etc/nginx/conf.d/*.conf; - js_import /usr/lib/nginx/modules/njs/httpmatches.js; - server_names_hash_bucket_size 256; - server_names_hash_max_size 1024; - } ---- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/manifests/nginx-conf.yaml b/deploy/manifests/nginx-conf.yaml new file mode 100644 index 000000000..ddb49f42d --- /dev/null +++ b/deploy/manifests/nginx-conf.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf + namespace: nginx-gateway +data: + nginx.conf: | + load_module /usr/lib/nginx/modules/ngx_http_js_module.so; + + events {} + + pid /etc/nginx/nginx.pid; + error_log stderr debug; + + http { + include /etc/nginx/conf.d/*.conf; + js_import /usr/lib/nginx/modules/njs/httpmatches.js; + server_names_hash_bucket_size 256; + server_names_hash_max_size 1024; + } diff --git a/deploy/manifests/rbac.yaml b/deploy/manifests/rbac.yaml new file mode 100644 index 000000000..068f3acda --- /dev/null +++ b/deploy/manifests/rbac.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nginx-gateway + namespace: nginx-gateway +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - services + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + - httproutes + verbs: + - list + - watch +- apiGroups: + - gateway.nginx.org + resources: + - gatewayconfigs + verbs: + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + - gateways/status + - gatewayclasses/status + verbs: + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nginx-gateway +subjects: +- kind: ServiceAccount + name: nginx-gateway + namespace: nginx-gateway +roleRef: + kind: ClusterRole + name: nginx-gateway + apiGroup: rbac.authorization.k8s.io diff --git a/docs/installation.md b/docs/installation.md index 1889c68a5..75e65bf7a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -35,6 +35,18 @@ This guide walks you through how to install NGINX Kubernetes Gateway on a generi kubectl create configmap njs-modules --from-file=internal/nginx/modules/src/httpmatches.js -n nginx-gateway ``` +1. Create the ConfigMap with the main NGINX configuration file: + + ``` + kubectl apply -f deploy/manifests/nginx-conf.yaml + ``` + +1. Configure RBAC: + + ``` + kubectl apply -f deploy/manifests/rbac.yaml + ``` + 1. Create the GatewayClass resource: ``` @@ -44,7 +56,7 @@ This guide walks you through how to install NGINX Kubernetes Gateway on a generi 1. Deploy the NGINX Kubernetes Gateway: ``` - kubectl apply -f deploy/manifests/nginx-gateway.yaml + kubectl apply -f deploy/manifests/deployment.yaml ``` 1. Confirm the NGINX Kubernetes Gateway is running in `nginx-gateway` namespace: diff --git a/embedded.go b/embedded.go new file mode 100644 index 000000000..2edc141b1 --- /dev/null +++ b/embedded.go @@ -0,0 +1,11 @@ +package embeddedfiles + +import _ "embed" + +// StaticModeDeploymentYAML contains the YAML manifest of the Deployment resource for the static mode. +// +// We put this in the root of the repo because goembed doesn't support relative/absolute paths and symlinks, +// and we want to keep the manifests in the deploy/manifests directory. +// +//go:embed deploy/manifests/deployment.yaml +var StaticModeDeploymentYAML []byte diff --git a/internal/helpers/helpers.go b/internal/helpers/helpers.go index 1a243950e..2d7c506b6 100644 --- a/internal/helpers/helpers.go +++ b/internal/helpers/helpers.go @@ -2,7 +2,10 @@ package helpers import ( + "fmt" + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/gateway-api/apis/v1beta1" ) @@ -62,3 +65,21 @@ func GetBoolPointer(b bool) *bool { func GetPointer[T any](v T) *T { return &v } + +// PrepareTimeForFakeClient processes the time similarly to the fake client +// from sigs.k8s.io/controller-runtime/pkg/client/fake +// making it is possible to use it in tests when comparing against values returned by the fake client. +// It panics if it fails to process the time. +func PrepareTimeForFakeClient(t metav1.Time) metav1.Time { + bytes, err := t.Marshal() + if err != nil { + panic(fmt.Errorf("failed to marshal time: %w", err)) + } + + err = t.Unmarshal(bytes) + if err != nil { + panic(fmt.Errorf("failed to unmarshal time: %w", err)) + } + + return t +} diff --git a/internal/provisioner/deployment.go b/internal/provisioner/deployment.go new file mode 100644 index 000000000..376c134df --- /dev/null +++ b/internal/provisioner/deployment.go @@ -0,0 +1,33 @@ +package provisioner + +import ( + "fmt" + + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" +) + +// prepareDeployment prepares a new the static mode Deployment based on the YAML manifest. +// It will use the specified id to set unique parts of the deployment, so it must be unique among all Deployments for +// Gateways. +// It will configure the Deployment to use the Gateway with the given NamespacedName. +func prepareDeployment(depYAML []byte, id string, gwNsName types.NamespacedName) (*v1.Deployment, error) { + dep := &v1.Deployment{} + err := yaml.Unmarshal(depYAML, dep) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal deployment: %w", err) + } + + dep.ObjectMeta.Name = id + dep.Spec.Selector.MatchLabels["app"] = id + dep.Spec.Template.ObjectMeta.Labels["app"] = id + + extraArgs := []string{ + "--gateway=" + gwNsName.String(), + "--update-gatewayclass-status=false", + } + dep.Spec.Template.Spec.Containers[0].Args = append(dep.Spec.Template.Spec.Containers[0].Args, extraArgs...) + + return dep, nil +} diff --git a/internal/provisioner/handler.go b/internal/provisioner/handler.go new file mode 100644 index 000000000..af0ff059b --- /dev/null +++ b/internal/provisioner/handler.go @@ -0,0 +1,136 @@ +package provisioner + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nginxinc/nginx-kubernetes-gateway/internal/events" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/state/conditions" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/status" +) + +// eventHandler ensures each Gateway for the specific GatewayClass has a corresponding Deployment +// of NKG configured to use that specific Gateway. +// +// eventHandler implements events.Handler interface. +type eventHandler struct { + gcName string + store *store + + // provisions maps NamespacedName of Gateway to its corresponding Deployment + provisions map[types.NamespacedName]*v1.Deployment + + statusUpdater status.Updater + k8sClient client.Client + logger logr.Logger + + staticModeDeploymentYAML []byte +} + +func newEventHandler( + gcName string, + statusUpdater status.Updater, + k8sClient client.Client, + logger logr.Logger, + staticModeDeploymentYAML []byte, +) *eventHandler { + return &eventHandler{ + store: newStore(), + provisions: make(map[types.NamespacedName]*v1.Deployment), + statusUpdater: statusUpdater, + gcName: gcName, + k8sClient: k8sClient, + logger: logger, + staticModeDeploymentYAML: staticModeDeploymentYAML, + } +} + +func (h *eventHandler) ensureGatewayClassAccepted(ctx context.Context) { + gc, exist := h.store.gatewayClasses[types.NamespacedName{Name: h.gcName}] + if !exist { + panic(fmt.Errorf("GatewayClass %s must exist", h.gcName)) + } + + statuses := status.Statuses{ + GatewayClassStatus: &status.GatewayClassStatus{ + Conditions: conditions.NewDefaultGatewayClassConditions(), + ObservedGeneration: gc.Generation, + }, + } + + h.statusUpdater.Update(ctx, statuses) +} + +func (h *eventHandler) ensureDeploymentsMatchGateways(ctx context.Context) { + var gwsWithoutDeps, removedGwsWithDeps []types.NamespacedName + + for nsname, gw := range h.store.gateways { + if string(gw.Spec.GatewayClassName) != h.gcName { + continue + } + if _, exist := h.provisions[nsname]; exist { + continue + } + + gwsWithoutDeps = append(gwsWithoutDeps, nsname) + } + + for nsname := range h.provisions { + if _, exist := h.store.gateways[nsname]; exist { + continue + } + + removedGwsWithDeps = append(removedGwsWithDeps, nsname) + } + + // Create new deployments + + for _, nsname := range gwsWithoutDeps { + deployment, err := prepareDeployment(h.staticModeDeploymentYAML, generateDeploymentID(nsname), nsname) + if err != nil { + panic(fmt.Errorf("failed to prepare deployment: %w", err)) + } + + err = h.k8sClient.Create(ctx, deployment) + if err != nil { + panic(fmt.Errorf("failed to create deployment: %w", err)) + } + + h.provisions[nsname] = deployment + + h.logger.Info("Created deployment", "deployment", client.ObjectKeyFromObject(deployment)) + } + + // Remove unnecessary deployments + + for _, nsname := range removedGwsWithDeps { + deployment := h.provisions[nsname] + + err := h.k8sClient.Delete(ctx, deployment) + if err != nil { + panic(fmt.Errorf("failed to delete deployment: %w", err)) + } + + delete(h.provisions, nsname) + + h.logger.Info("Deleted deployment", "deployment", client.ObjectKeyFromObject(deployment)) + } +} + +func (h *eventHandler) HandleEventBatch(ctx context.Context, batch events.EventBatch) { + h.store.update(batch) + h.ensureGatewayClassAccepted(ctx) + h.ensureDeploymentsMatchGateways(ctx) +} + +func generateDeploymentID(gatewayNsName types.NamespacedName) string { + // for production, make sure the ID is: + // - a valid resource name (ex. can't be too long); + // - unique among all Gateway resources (Gateways test-test/test and test/test-test should not have the same ID) + return fmt.Sprintf("nginx-gateway-%s-%s", gatewayNsName.Namespace, gatewayNsName.Name) +} diff --git a/internal/provisioner/handler_test.go b/internal/provisioner/handler_test.go new file mode 100644 index 000000000..0df58c337 --- /dev/null +++ b/internal/provisioner/handler_test.go @@ -0,0 +1,370 @@ +package provisioner + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/gateway-api/apis/v1beta1" + + . "github.com/onsi/gomega" + + embeddedfiles "github.com/nginxinc/nginx-kubernetes-gateway" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/events" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/helpers" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/status" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/status/statusfakes" +) + +var _ = Describe("handler", func() { + const ( + gcName = "test-gc" + ) + var ( + handler *eventHandler + fakeClockTime metav1.Time + + statusUpdater status.Updater + k8sclient client.Client + + gwNsName, depNsName types.NamespacedName + gw *v1beta1.Gateway + ) + + BeforeEach(OncePerOrdered, func() { + scheme := runtime.NewScheme() + + Expect(v1beta1.AddToScheme(scheme)).Should(Succeed()) + Expect(v1.AddToScheme(scheme)).Should(Succeed()) + + k8sclient = fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource( + &v1beta1.Gateway{}, + &v1beta1.GatewayClass{}, + ). + Build() + + fakeClockTime = helpers.PrepareTimeForFakeClient(metav1.Now()) + fakeClock := &statusfakes.FakeClock{} + fakeClock.NowReturns(fakeClockTime) + + statusUpdater = status.NewUpdater(status.UpdaterConfig{ + Client: k8sclient, + Clock: fakeClock, + Logger: zap.New(), + GatewayCtlrName: "test.example.com", + GatewayClassName: gcName, + PodIP: "1.2.3.4", + UpdateGatewayClassStatus: true, + }) + + gwNsName = types.NamespacedName{ + Namespace: "test-ns", + Name: "test-gw", + } + gw = &v1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: gwNsName.Namespace, + Name: gwNsName.Name, + }, + Spec: v1beta1.GatewaySpec{ + GatewayClassName: gcName, + }, + } + + depNsName = types.NamespacedName{ + Namespace: "nginx-gateway", + Name: "nginx-gateway-test-ns-test-gw", + } + }) + + itShouldUpsertGatewayClass := func() { + // Add GatewayClass to the cluster + + gc := &v1beta1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: gcName, + }, + } + + err := k8sclient.Create(context.Background(), gc) + Expect(err).ShouldNot(HaveOccurred()) + + // UpsertGatewayClass + + batch := []interface{}{ + &events.UpsertEvent{ + Resource: gc, + }, + } + handler.HandleEventBatch(context.Background(), batch) + + // Ensure GatewayClass is accepted + + clusterGc := &v1beta1.GatewayClass{} + err = k8sclient.Get(context.Background(), client.ObjectKeyFromObject(gc), clusterGc) + + Expect(err).ShouldNot(HaveOccurred()) + + expectedConditions := []metav1.Condition{ + { + Type: string(v1beta1.GatewayClassConditionStatusAccepted), + Status: metav1.ConditionTrue, + ObservedGeneration: 0, + LastTransitionTime: fakeClockTime, + Reason: "Accepted", + Message: "GatewayClass is accepted", + }, + } + + Expect(clusterGc.Status.Conditions).To(Equal(expectedConditions)) + } + + itShouldUpsertGateway := func() { + batch := []interface{}{ + &events.UpsertEvent{ + Resource: gw, + }, + } + + handler.HandleEventBatch(context.Background(), batch) + + dep := &v1.Deployment{} + err := k8sclient.Get(context.Background(), depNsName, dep) + + Expect(err).ShouldNot(HaveOccurred()) + + Expect(dep.ObjectMeta.Namespace).To(Equal("nginx-gateway")) + Expect(dep.ObjectMeta.Name).To(Equal("nginx-gateway-test-ns-test-gw")) + Expect(dep.Spec.Template.Spec.Containers[0].Args).To(ContainElement("static-mode")) + Expect(dep.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--gateway=test-ns/test-gw")) + Expect(dep.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--update-gatewayclass-status=false")) + } + + itShouldPanicWhenUpsertingGateway := func() { + batch := []interface{}{ + &events.UpsertEvent{ + Resource: gw, + }, + } + + handle := func() { + handler.HandleEventBatch(context.Background(), batch) + } + + Expect(handle).Should(Panic()) + } + + Describe("Core cases", Ordered, func() { + BeforeAll(func() { + handler = newEventHandler( + gcName, + statusUpdater, + k8sclient, + zap.New(), + embeddedfiles.StaticModeDeploymentYAML, + ) + }) + + When("upserting GatewayClass", func() { + It("should make GatewayClass Accepted", func() { + itShouldUpsertGatewayClass() + }) + }) + + When("upserting Gateway", func() { + It("should create Deployment", func() { + itShouldUpsertGateway() + }) + }) + + When("upserting Gateway again", func() { + It("must retain Deployment", func() { + itShouldUpsertGateway() + }) + }) + + When("deleting Gateway", func() { + It("should remove Deployment", func() { + batch := []interface{}{ + &events.DeleteEvent{ + Type: &v1beta1.Gateway{}, + NamespacedName: gwNsName, + }, + } + + handler.HandleEventBatch(context.Background(), batch) + + deps := &v1.DeploymentList{} + + err := k8sclient.List(context.Background(), deps) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(deps.Items).To(HaveLen(0)) + }) + }) + + When("upserting Gateway for a different GatewayClass", func() { + It("should not create Deployment", func() { + gw := &v1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw-2", + Namespace: "test-ns-2", + }, + Spec: v1beta1.GatewaySpec{ + GatewayClassName: "some-class", + }, + } + + batch := []interface{}{ + &events.UpsertEvent{ + Resource: gw, + }, + } + + handler.HandleEventBatch(context.Background(), batch) + + deps := &v1.DeploymentList{} + err := k8sclient.List(context.Background(), deps) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(deps.Items).To(HaveLen(0)) + }) + }) + }) + + Describe("Edge cases", func() { + BeforeEach(func() { + handler = newEventHandler( + gcName, + statusUpdater, + k8sclient, + zap.New(), + embeddedfiles.StaticModeDeploymentYAML, + ) + }) + + DescribeTable("Edge cases for events", + func(e interface{}) { + batch := []interface{}{e} + + handle := func() { + handler.HandleEventBatch(context.TODO(), batch) + } + + Expect(handle).Should(Panic()) + }, + Entry("should panic for an unknown event type", + &struct{}{}), + Entry("should panic for an unknown type of resource in upsert event", + &events.UpsertEvent{ + Resource: &v1beta1.HTTPRoute{}, + }), + Entry("should panic for an unknown type of resource in delete event", + &events.DeleteEvent{ + Type: &v1beta1.HTTPRoute{}, + }), + ) + + When("upserting Gateway when GatewayClass doesn't exist", func() { + It("should panic", func() { + itShouldPanicWhenUpsertingGateway() + }) + }) + + When("upserting Gateway when Deployment can't be created", func() { + It("should panic", func() { + itShouldUpsertGatewayClass() + + // Create a deployment so that the Handler will fail to create it because it already exists. + + dep := &v1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: depNsName.Namespace, + Name: depNsName.Name, + }, + } + + err := k8sclient.Create(context.Background(), dep) + Expect(err).ShouldNot(HaveOccurred()) + + itShouldPanicWhenUpsertingGateway() + }) + }) + + When("deleting Gateway when Deployment can't be deleted", func() { + It("should panic", func() { + itShouldUpsertGatewayClass() + itShouldUpsertGateway() + + // Delete the deployment so that the Handler will fail to delete it because it doesn't exist. + + dep := &v1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: depNsName.Namespace, + Name: depNsName.Name, + }, + } + + err := k8sclient.Delete(context.Background(), dep) + Expect(err).ShouldNot(HaveOccurred()) + + batch := []interface{}{ + &events.DeleteEvent{ + Type: &v1beta1.Gateway{}, + NamespacedName: gwNsName, + }, + } + + handle := func() { + handler.HandleEventBatch(context.Background(), batch) + } + + Expect(handle).Should(Panic()) + }) + }) + + When("deleting GatewayClass", func() { + It("should panic", func() { + itShouldUpsertGatewayClass() + + batch := []interface{}{ + &events.DeleteEvent{ + Type: &v1beta1.GatewayClass{}, + NamespacedName: types.NamespacedName{ + Name: gcName, + }, + }, + } + + handle := func() { + handler.HandleEventBatch(context.Background(), batch) + } + + Expect(handle).Should(Panic()) + }) + }) + }) + + When("upserting Gateway with broken static Deployment YAML", func() { + It("it should panic", func() { + handler = newEventHandler( + gcName, + statusUpdater, + k8sclient, + zap.New(), + []byte("broken YAML"), + ) + + itShouldUpsertGatewayClass() + itShouldPanicWhenUpsertingGateway() + }) + }) +}) diff --git a/internal/provisioner/manager.go b/internal/provisioner/manager.go new file mode 100644 index 000000000..d394ad05c --- /dev/null +++ b/internal/provisioner/manager.go @@ -0,0 +1,125 @@ +package provisioner + +import ( + "fmt" + + "github.com/go-logr/logr" + v1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + ctlr "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + embeddedfiles "github.com/nginxinc/nginx-kubernetes-gateway" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/controller" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/events" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/manager/filter" + "github.com/nginxinc/nginx-kubernetes-gateway/internal/status" +) + +// Config is configuration for the provisioner mode. +type Config struct { + Logger logr.Logger + GatewayClassName string +} + +// StartManager starts a Manager for the provisioner mode, which provisions +// a Deployment of NKG (static mode) for each Gateway of the provisioner GatewayClass. +// +// The provisioner mode is introduced to allow running Gateway API conformance tests for NKG, which expects +// an independent data plane instance being provisioned for each Gateway. +// +// The provisioner mode is not intended to be used in production (in the short term), as it lacks support for +// many important features. See https://github.com/nginxinc/nginx-kubernetes-gateway/issues/634 for more details. +func StartManager(cfg Config) error { + scheme := runtime.NewScheme() + utilruntime.Must(gatewayv1beta1.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + + options := manager.Options{ + Scheme: scheme, + Logger: cfg.Logger, + } + clusterCfg := ctlr.GetConfigOrDie() + + mgr, err := manager.New(clusterCfg, options) + if err != nil { + return fmt.Errorf("cannot build runtime manager: %w", err) + } + + // Note: for any new object type or a change to the existing one, + // make sure to also update firstBatchPreparer creation below + controllerRegCfgs := []struct { + objectType client.Object + options []controller.Option + }{ + { + objectType: &gatewayv1beta1.GatewayClass{}, + options: []controller.Option{ + controller.WithNamespacedNameFilter( + filter.CreateSingleResourceFilter(types.NamespacedName{Name: cfg.GatewayClassName}), + ), + }, + }, + { + objectType: &gatewayv1beta1.Gateway{}, + }, + } + + ctx := ctlr.SetupSignalHandler() + eventCh := make(chan interface{}) + + for _, regCfg := range controllerRegCfgs { + err := controller.Register(ctx, regCfg.objectType, mgr, eventCh, regCfg.options...) + if err != nil { + return fmt.Errorf("cannot register controller for %T: %w", regCfg.objectType, err) + } + } + + firstBatchPreparer := events.NewFirstEventBatchPreparerImpl( + mgr.GetCache(), + []client.Object{ + &gatewayv1beta1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: cfg.GatewayClassName}}, + }, + []client.ObjectList{ + &gatewayv1beta1.GatewayList{}, + }, + ) + + statusUpdater := status.NewUpdater( + status.UpdaterConfig{ + Client: mgr.GetClient(), + Clock: status.NewRealClock(), + Logger: cfg.Logger.WithName("statusUpdater"), + GatewayClassName: cfg.GatewayClassName, + UpdateGatewayClassStatus: true, + }, + ) + + handler := newEventHandler( + cfg.GatewayClassName, + statusUpdater, + mgr.GetClient(), + cfg.Logger.WithName("eventHandler"), + embeddedfiles.StaticModeDeploymentYAML, + ) + + eventLoop := events.NewEventLoop( + eventCh, + cfg.Logger.WithName("eventLoop"), + handler, + firstBatchPreparer, + ) + + err = mgr.Add(eventLoop) + if err != nil { + return fmt.Errorf("cannot register event loop: %w", err) + } + + cfg.Logger.Info("Starting manager") + return mgr.Start(ctx) +} diff --git a/internal/provisioner/provisioner_suite_test.go b/internal/provisioner/provisioner_suite_test.go new file mode 100644 index 000000000..6cfdb4144 --- /dev/null +++ b/internal/provisioner/provisioner_suite_test.go @@ -0,0 +1,13 @@ +package provisioner + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestProvisioner(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Provisioner Suite") +} diff --git a/internal/provisioner/store.go b/internal/provisioner/store.go new file mode 100644 index 000000000..f13e57b16 --- /dev/null +++ b/internal/provisioner/store.go @@ -0,0 +1,51 @@ +package provisioner + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/nginxinc/nginx-kubernetes-gateway/internal/events" +) + +// store stores the cluster state needed by the provisioner and allows to update it from the events. +type store struct { + gatewayClasses map[types.NamespacedName]*v1beta1.GatewayClass + gateways map[types.NamespacedName]*v1beta1.Gateway +} + +func newStore() *store { + return &store{ + gatewayClasses: make(map[types.NamespacedName]*v1beta1.GatewayClass), + gateways: make(map[types.NamespacedName]*v1beta1.Gateway), + } +} + +func (s *store) update(batch events.EventBatch) { + for _, event := range batch { + switch e := event.(type) { + case *events.UpsertEvent: + switch obj := e.Resource.(type) { + case *v1beta1.GatewayClass: + s.gatewayClasses[client.ObjectKeyFromObject(obj)] = obj + case *v1beta1.Gateway: + s.gateways[client.ObjectKeyFromObject(obj)] = obj + default: + panic(fmt.Errorf("unknown resource type %T", e.Resource)) + } + case *events.DeleteEvent: + switch e.Type.(type) { + case *v1beta1.GatewayClass: + delete(s.gatewayClasses, e.NamespacedName) + case *v1beta1.Gateway: + delete(s.gateways, e.NamespacedName) + default: + panic(fmt.Errorf("unknown resource type %T", e.Type)) + } + default: + panic(fmt.Errorf("unknown event type %T", e)) + } + } +} diff --git a/internal/status/updater_test.go b/internal/status/updater_test.go index 43e48e28b..f1020c196 100644 --- a/internal/status/updater_test.go +++ b/internal/status/updater_test.go @@ -45,10 +45,7 @@ var _ = Describe("Updater", func() { ). Build() - // Rfc3339Copy() removes the monotonic clock reading and leaves only second-level precision. - // We use it because updating the status in the FakeClient and then getting the resource back - // involves encoding and decoding the resource to/from JSON, which uses RFC 3339 for metav1.Time. - fakeClockTime = metav1.NewTime(time.Now()).Rfc3339Copy() + fakeClockTime = helpers.PrepareTimeForFakeClient(metav1.NewTime(time.Now())) fakeClock = &statusfakes.FakeClock{} fakeClock.NowReturns(fakeClockTime)