diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 02bd1eaf03..4aae90c185 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -135,6 +135,12 @@ jobs: kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }} kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }} + - name: Setup license file for plus + if: ${{ inputs.image == 'plus' }} + run: echo "$PLUS_LICENSE" > license.jwt + env: + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }} + - name: Setup conformance tests run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 2dd7388fc4..d3b47a269d 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -100,6 +100,12 @@ jobs: NGINX_CONF_DIR=internal/mode/static/nginx/conf BUILD_AGENT=gha + - name: Setup license file for plus + if: ${{ inputs.image == 'plus' }} + run: echo "$PLUS_LICENSE" > license.jwt + env: + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }} + - name: Install cloud-provider-kind run: | CLOUD_PROVIDER_KIND_VERSION=v0.4.0 # renovate: datasource=github-tags depName=kubernetes-sigs/cloud-provider-kind diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 2292d06b2b..3fd7f4d725 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -143,10 +143,14 @@ jobs: kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f - kubectl create namespace nginx-gateway - - name: Create k8s secret + - name: Create plus secrets if: ${{ inputs.image == 'plus' }} + env: + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }} run: | + echo "$PLUS_LICENSE" > license.jwt kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username=${{ secrets.JWT_PLUS_REGISTRY }} --docker-password=none -n nginx-gateway + kubectl create secret generic nplus-license --from-file license.jwt -n nginx-gateway - name: Set up Python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index f4ce735d05..6a9bdd1007 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -111,6 +111,12 @@ jobs: echo "GKE_NUM_NODES=12" >> vars.env echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env + - name: Setup license file for plus + if: matrix.type == 'plus' + run: echo "$PLUS_LICENSE" > license.jwt + env: + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }} + - name: Create GKE cluster working-directory: ./tests run: make create-gke-cluster CI=true diff --git a/.gitignore b/.gitignore index 3615727c6d..4dbda581d7 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,9 @@ internal/mode/static/nginx/modules/coverage *.crt *.key +# JWT files +*.jwt + # Dotenv files **/*.env diff --git a/.hugo_build.lock b/.hugo_build.lock deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.yamllint.yaml b/.yamllint.yaml index f4ae917a19..b2d07c848f 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -3,6 +3,7 @@ ignore: - charts/nginx-gateway-fabric/templates - config/crd/bases/ - deploy/crds.yaml + - deploy/*nginx-plus - site/static rules: diff --git a/Makefile b/Makefile index cf939238f0..ed68d2b7f6 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,12 @@ # variables that should not be overridden by the user VERSION = edge -SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +SELF_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) CHART_DIR = $(SELF_DIR)charts/nginx-gateway-fabric NGINX_CONF_DIR = internal/mode/static/nginx/conf NJS_DIR = internal/mode/static/nginx/modules/src KIND_CONFIG_FILE = $(SELF_DIR)config/cluster/kind-cluster.yaml -NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key -BUILD_AGENT=local -PLUS_ENABLED ?= false +NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=$(SELF_DIR)nginx-repo.crt --secret id=nginx-repo.key,src=$(SELF_DIR)nginx-repo.key +BUILD_AGENT = local PROD_TELEMETRY_ENDPOINT = oss.edge.df.f5.com:443 # the telemetry related variables below are also configured in goreleaser.yml @@ -49,6 +48,8 @@ TARGET ?= local## The target of the build. Possible values: local and container OUT_DIR ?= build/out## The folder where the binary will be stored GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64 GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin +PLUS_ENABLED ?= false +PLUS_LICENSE_FILE ?= $(SELF_DIR)license.jwt override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) --build-arg BUILD_AGENT=$(BUILD_AGENT) .DEFAULT_GOAL := help @@ -227,7 +228,9 @@ helm-install-local: install-gateway-crds ## Helm install NGF on configured kind .PHONY: helm-install-local-with-plus helm-install-local-with-plus: install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus. - helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --create-namespace --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS) + kubectl create namespace nginx-gateway || true + kubectl -n nginx-gateway create secret generic nplus-license --from-file $(PLUS_LICENSE_FILE) || true + helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS) # Debug Targets .PHONY: debug-build diff --git a/build/Dockerfile.nginxplus b/build/Dockerfile.nginxplus index 2d2d3c249a..192f7839f3 100644 --- a/build/Dockerfile.nginxplus +++ b/build/Dockerfile.nginxplus @@ -7,7 +7,7 @@ ADD --link --chown=101:1001 https://cs.nginx.com/static/keys/nginx_signing.rsa.p FROM alpine:3.20 -ARG NGINX_PLUS_VERSION=R32 +ARG NGINX_PLUS_VERSION=R33 ARG NJS_DIR ARG NGINX_CONF_DIR ARG BUILD_AGENT diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index b8c6a35b3e..27a4876883 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -268,10 +268,12 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginx.image.tag` | | string | `"edge"` | | `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` | | `nginx.plus` | Is NGINX Plus image being used | bool | `false` | -| `nginx.usage.clusterName` | The display name of the Kubernetes cluster in the NGINX Plus usage reporting server. | string | `""` | -| `nginx.usage.insecureSkipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` | -| `nginx.usage.secretName` | The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting. | string | `""` | -| `nginx.usage.serverURL` | The base server URL of the NGINX Plus usage reporting server. | string | `""` | +| `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` | +| `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` | +| `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` | +| `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` | +| `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` | +| `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` | | `nginxGateway.config.logging.level` | Log level. | string | `"info"` | | `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` | | `nginxGateway.extraVolumeMounts` | extraVolumeMounts are the additional volume mounts for the nginx-gateway container. | list | `[]` | diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index e43d1e76cb..cbb163ae1d 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -18,7 +18,7 @@ rules: - get - list - watch -{{- if .Values.nginxGateway.productTelemetry.enable }} +{{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }} - apiGroups: - "" resources: diff --git a/charts/nginx-gateway-fabric/templates/configmap.yaml b/charts/nginx-gateway-fabric/templates/configmap.yaml new file mode 100644 index 0000000000..69586d5ec3 --- /dev/null +++ b/charts/nginx-gateway-fabric/templates/configmap.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-includes-bootstrap + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-gateway.labels" . | nindent 4 }} +data: + main.conf: | + {{- if and .Values.nginx.config .Values.nginx.config.logging .Values.nginx.config.logging.errorLevel }} + error_log stderr {{ .Values.nginx.config.logging.errorLevel }}; + {{ else }} + error_log stderr info; + {{- end }} + {{- if .Values.nginx.plus }} + mgmt.conf: | + mgmt { + {{- if .Values.nginx.usage.endpoint }} + usage_report endpoint={{ .Values.nginx.usage.endpoint }}; + {{- end }} + {{- if .Values.nginx.usage.skipVerify }} + ssl_verify off; + {{- end }} + {{- if .Values.nginx.usage.caSecretName }} + ssl_trusted_certificate /etc/nginx/certs-bootstrap/ca.crt; + {{- end }} + {{- if .Values.nginx.usage.clientSSLSecretName }} + ssl_certificate /etc/nginx/certs-bootstrap/tls.crt; + ssl_certificate_key /etc/nginx/certs-bootstrap/tls.key; + {{- end }} + enforce_initial_report off; + } + {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml index d1bd9e8b05..92ee1eb7c3 100644 --- a/charts/nginx-gateway-fabric/templates/deployment.yaml +++ b/charts/nginx-gateway-fabric/templates/deployment.yaml @@ -42,8 +42,12 @@ spec: - copy - --source - /includes/main.conf + {{- if .Values.nginx.plus }} + - --source + - /includes/mgmt.conf + {{- end }} - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes securityContext: seccompProfile: type: RuntimeDefault @@ -56,7 +60,7 @@ spec: runAsUser: 102 runAsGroup: 1001 volumeMounts: - - name: nginx-includes-configmap + - name: nginx-includes-bootstrap mountPath: /includes - name: nginx-main-includes mountPath: /etc/nginx/main-includes @@ -69,6 +73,24 @@ spec: - --service={{ include "nginx-gateway.fullname" . }} {{- if .Values.nginx.plus }} - --nginx-plus + {{- if .Values.nginx.usage.secretName }} + - --usage-report-secret={{ .Values.nginx.usage.secretName }} + {{- end }} + {{- if .Values.nginx.usage.endpoint }} + - --usage-report-endpoint={{ .Values.nginx.usage.endpoint }} + {{- end }} + {{- if .Values.nginx.usage.resolver }} + - --usage-report-resolver={{ .Values.nginx.usage.resolver }} + {{- end }} + {{- if .Values.nginx.usage.skipVerify }} + - --usage-report-skip-verify + {{- end }} + {{- if .Values.nginx.usage.caSecretName }} + - --usage-report-ca-secret={{ .Values.nginx.usage.caSecretName }} + {{- end }} + {{- if .Values.nginx.usage.clientSSLSecretName }} + - --usage-report-client-ssl-secret={{ .Values.nginx.usage.clientSSLSecretName }} + {{- end }} {{- end }} {{- if .Values.metrics.enable }} - --metrics-port={{ .Values.metrics.port }} @@ -94,18 +116,6 @@ spec: {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }} - --gateway-api-experimental-features {{- end }} - {{- if .Values.nginx.usage.secretName }} - - --usage-report-secret={{ .Values.nginx.usage.secretName }} - {{- end }} - {{- if .Values.nginx.usage.serverURL }} - - --usage-report-server-url={{ .Values.nginx.usage.serverURL }} - {{- end }} - {{- if .Values.nginx.usage.clusterName }} - - --usage-report-cluster-name={{ .Values.nginx.usage.clusterName }} - {{- end }} - {{- if .Values.nginx.usage.insecureSkipVerify }} - - --usage-report-skip-verify - {{- end }} {{- if .Values.nginxGateway.snippetsFilters.enable }} - --snippets-filters {{- end }} @@ -212,8 +222,21 @@ spec: mountPath: /var/run/nginx - name: nginx-cache mountPath: /var/cache/nginx + - name: nginx-lib + mountPath: /var/lib/nginx - name: nginx-includes mountPath: /etc/nginx/includes + {{- if .Values.nginx.plus }} + {{- if .Values.nginx.usage.secretName }} + - name: nginx-plus-license + mountPath: /etc/nginx/license.jwt + subPath: license.jwt + {{- end }} + {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }} + - name: nginx-plus-usage-certs + mountPath: /etc/nginx/certs-bootstrap/ + {{- end }} + {{- end }} {{- with .Values.nginx.extraVolumeMounts -}} {{ toYaml . | nindent 8 }} {{- end }} @@ -255,11 +278,33 @@ spec: emptyDir: {} - name: nginx-cache emptyDir: {} + - name: nginx-lib + emptyDir: {} - name: nginx-includes emptyDir: {} - - name: nginx-includes-configmap + - name: nginx-includes-bootstrap configMap: - name: nginx-includes + name: nginx-includes-bootstrap + {{- if .Values.nginx.plus }} + {{- if .Values.nginx.usage.secretName }} + - name: nginx-plus-license + secret: + secretName: {{ .Values.nginx.usage.secretName }} + {{- end }} + {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }} + - name: nginx-plus-usage-certs + projected: + sources: + {{- if .Values.nginx.usage.caSecretName }} + - secret: + name: {{ .Values.nginx.usage.caSecretName }} + {{- end }} + {{- if .Values.nginx.usage.clientSSLSecretName }} + - secret: + name: {{ .Values.nginx.usage.clientSSLSecretName }} + {{- end }} + {{- end }} + {{- end }} {{- with .Values.extraVolumes -}} {{ toYaml . | nindent 6 }} {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/include-configmap.yaml b/charts/nginx-gateway-fabric/templates/include-configmap.yaml deleted file mode 100644 index 9321861c2d..0000000000 --- a/charts/nginx-gateway-fabric/templates/include-configmap.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx-includes - namespace: {{ .Release.Namespace }} - labels: - {{- include "nginx-gateway.labels" . | nindent 4 }} -data: - main.conf: | - {{- if and .Values.nginx.config .Values.nginx.config.logging .Values.nginx.config.logging.errorLevel }} - error_log stderr {{ .Values.nginx.config.logging.errorLevel }}; - {{ else }} - error_log stderr info; - {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/scc.yaml b/charts/nginx-gateway-fabric/templates/scc.yaml index b156ff2109..8156b279b7 100644 --- a/charts/nginx-gateway-fabric/templates/scc.yaml +++ b/charts/nginx-gateway-fabric/templates/scc.yaml @@ -33,6 +33,7 @@ volumes: - emptyDir - secret - configMap +- projected users: - {{ printf "system:serviceaccount:%s:%s" .Release.Namespace (include "nginx-gateway.serviceAccountName" .) }} allowedCapabilities: diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 38cbbd34d2..a83d54c431 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -262,33 +262,47 @@ "usage": { "description": "Configuration for NGINX Plus usage reporting.", "properties": { - "clusterName": { + "caSecretName": { "default": "", - "description": "The display name of the Kubernetes cluster in the NGINX Plus usage reporting server.", + "description": "The name of the Secret containing the NGINX Instance Manager CA certificate.\nMust exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).", "required": [], - "title": "clusterName", + "title": "caSecretName", "type": "string" }, - "insecureSkipVerify": { - "default": false, - "description": "Disable client verification of the NGINX Plus usage reporting server certificate.", + "clientSSLSecretName": { + "default": "", + "description": "The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager.\nMust exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).", "required": [], - "title": "insecureSkipVerify", - "type": "boolean" + "title": "clientSSLSecretName", + "type": "string" }, - "secretName": { + "endpoint": { "default": "", - "description": "The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting.", + "description": "The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com", "required": [], - "title": "secretName", + "title": "endpoint", "type": "string" }, - "serverURL": { + "resolver": { "default": "", - "description": "The base server URL of the NGINX Plus usage reporting server.", + "description": "The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager.", "required": [], - "title": "serverURL", + "title": "resolver", "type": "string" + }, + "secretName": { + "default": "nplus-license", + "description": "The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace\nthat the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).", + "required": [], + "title": "secretName", + "type": "string" + }, + "skipVerify": { + "default": false, + "description": "Disable client verification of the NGINX Plus usage reporting server certificate.", + "required": [], + "title": "skipVerify", + "type": "boolean" } }, "required": [], diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 3200a4eae9..073e4be531 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -134,6 +134,29 @@ nginx: # -- Is NGINX Plus image being used plus: false + # Configuration for NGINX Plus usage reporting. + usage: + # -- The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace + # that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + secretName: "nplus-license" + + # -- The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com + endpoint: "" + + # -- The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. + resolver: "" + + # -- Disable client verification of the NGINX Plus usage reporting server certificate. + skipVerify: false + + # -- The name of the Secret containing the NGINX Instance Manager CA certificate. + # Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + caSecretName: "" + + # -- The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. + # Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + clientSSLSecretName: "" + # @schema # type: object # properties: @@ -228,20 +251,6 @@ nginx: # -- Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. debug: false - # Configuration for NGINX Plus usage reporting. - usage: - # -- The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting. - secretName: "" - - # -- The base server URL of the NGINX Plus usage reporting server. - serverURL: "" - - # -- The display name of the Kubernetes cluster in the NGINX Plus usage reporting server. - clusterName: "" - - # -- Disable client verification of the NGINX Plus usage reporting server certificate. - insecureSkipVerify: false - # -- The lifecycle of the nginx container. lifecycle: {} diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index c365ff8eac..72b64db403 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "path/filepath" "runtime/debug" "strconv" "time" @@ -46,28 +47,31 @@ func createRootCommand() *cobra.Command { return rootCmd } +//nolint:gocyclo func createStaticModeCommand() *cobra.Command { // flag names const ( - gatewayFlag = "gateway" - configFlag = "config" - serviceFlag = "service" - updateGCStatusFlag = "update-gatewayclass-status" - metricsDisableFlag = "metrics-disable" - metricsSecureFlag = "metrics-secure-serving" - metricsPortFlag = "metrics-port" - healthDisableFlag = "health-disable" - healthPortFlag = "health-port" - leaderElectionDisableFlag = "leader-election-disable" - leaderElectionLockNameFlag = "leader-election-lock-name" - productTelemetryDisableFlag = "product-telemetry-disable" - plusFlag = "nginx-plus" - gwAPIExperimentalFlag = "gateway-api-experimental-features" - usageReportSecretFlag = "usage-report-secret" - usageReportServerURLFlag = "usage-report-server-url" - usageReportSkipVerifyFlag = "usage-report-skip-verify" - usageReportClusterNameFlag = "usage-report-cluster-name" - snippetsFiltersFlag = "snippets-filters" + gatewayFlag = "gateway" + configFlag = "config" + serviceFlag = "service" + updateGCStatusFlag = "update-gatewayclass-status" + metricsDisableFlag = "metrics-disable" + metricsSecureFlag = "metrics-secure-serving" + metricsPortFlag = "metrics-port" + healthDisableFlag = "health-disable" + healthPortFlag = "health-port" + leaderElectionDisableFlag = "leader-election-disable" + leaderElectionLockNameFlag = "leader-election-lock-name" + productTelemetryDisableFlag = "product-telemetry-disable" + plusFlag = "nginx-plus" + gwAPIExperimentalFlag = "gateway-api-experimental-features" + usageReportSecretFlag = "usage-report-secret" + usageReportEndpointFlag = "usage-report-endpoint" + usageReportResolverFlag = "usage-report-resolver" + usageReportSkipVerifyFlag = "usage-report-skip-verify" + usageReportClientSSLSecretFlag = "usage-report-client-ssl-secret" //nolint:gosec // not credentials + usageReportCASecretFlag = "usage-report-ca-secret" //nolint:gosec // not credentials + snippetsFiltersFlag = "snippets-filters" ) // flag values @@ -110,17 +114,26 @@ func createStaticModeCommand() *cobra.Command { disableProductTelemetry bool - plus bool - usageReportSkipVerify bool - usageReportClusterName = stringValidatingValue{ - validator: validateQualifiedName, + snippetsFilters bool + + plus bool + usageReportSkipVerify bool + usageReportSecretName = stringValidatingValue{ + validator: validateResourceName, + value: "nplus-license", } - usageReportSecretName = namespacedNameValue{} - usageReportServerURL = stringValidatingValue{ - validator: validateURL, + usageReportEndpoint = stringValidatingValue{ + validator: validateEndpointOptionalPort, + } + usageReportResolver = stringValidatingValue{ + validator: validateEndpointOptionalPort, + } + usageReportClientSSLSecretName = stringValidatingValue{ + validator: validateResourceName, + } + usageReportCASecretName = stringValidatingValue{ + validator: validateResourceName, } - - snippetsFilters bool ) cmd := &cobra.Command{ @@ -187,17 +200,19 @@ func createStaticModeCommand() *cobra.Command { gwNsName = &gateway.value } - var usageReportConfig *config.UsageReportConfig - if cmd.Flags().Changed(usageReportSecretFlag) { - if !plus { - return errors.New("usage-report arguments are only valid if using nginx-plus") - } + var usageReportConfig config.UsageReportConfig + if plus && usageReportSecretName.value == "" { + return errors.New("usage-report-secret is required when using NGINX Plus") + } - usageReportConfig = &config.UsageReportConfig{ - SecretNsName: usageReportSecretName.value, - ServerURL: usageReportServerURL.value, - ClusterDisplayName: usageReportClusterName.value, - InsecureSkipVerify: usageReportSkipVerify, + if plus { + usageReportConfig = config.UsageReportConfig{ + SecretName: usageReportSecretName.value, + ClientSSLSecretName: usageReportClientSSLSecretName.value, + CASecretName: usageReportCASecretName.value, + Endpoint: usageReportEndpoint.value, + Resolver: usageReportResolver.value, + SkipVerify: usageReportSkipVerify, } } @@ -378,21 +393,20 @@ func createStaticModeCommand() *cobra.Command { cmd.Flags().Var( &usageReportSecretName, usageReportSecretFlag, - "The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting.", + "The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace "+ + "that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).", ) cmd.Flags().Var( - &usageReportServerURL, - usageReportServerURLFlag, - "The base server URL of the NGINX Plus usage reporting server.", + &usageReportEndpoint, + usageReportEndpointFlag, + "The endpoint of the NGINX Plus usage reporting server.", ) - cmd.MarkFlagsRequiredTogether(usageReportSecretFlag, usageReportServerURLFlag) - cmd.Flags().Var( - &usageReportClusterName, - usageReportClusterNameFlag, - "The display name of the Kubernetes cluster in the NGINX Plus usage reporting server.", + &usageReportResolver, + usageReportResolverFlag, + "The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager.", ) cmd.Flags().BoolVar( @@ -402,6 +416,22 @@ func createStaticModeCommand() *cobra.Command { "Disable client verification of the NGINX Plus usage reporting server certificate.", ) + cmd.Flags().Var( + &usageReportClientSSLSecretName, + usageReportClientSSLSecretFlag, + "The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. "+ + "Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in "+ + "(default namespace: nginx-gateway).", + ) + + cmd.Flags().Var( + &usageReportCASecretName, + usageReportCASecretFlag, + "The name of the Secret containing the NGINX Instance Manager CA certificate. "+ + "Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in "+ + "(default namespace: nginx-gateway).", + ) + cmd.Flags().BoolVar( &snippetsFilters, snippetsFiltersFlag, @@ -499,51 +529,39 @@ func createCopyCommand() *cobra.Command { const srcFlag = "source" const destFlag = "destination" // flag values - var src, dest string + var srcFiles []string + var dest string cmd := &cobra.Command{ Use: "copy", - Short: "Copy a file to a destination", + Short: "Copy files to another directory", RunE: func(_ *cobra.Command, _ []string) error { - if len(src) == 0 { - return errors.New("source must not be empty") - } - if len(dest) == 0 { - return errors.New("destination must not be empty") + if err := validateSleepArgs(srcFiles, dest); err != nil { + return err } - srcFile, err := os.Open(src) - if err != nil { - return fmt.Errorf("error opening source file: %w", err) - } - defer srcFile.Close() - - destFile, err := os.Create(dest) - if err != nil { - return fmt.Errorf("error creating destination file: %w", err) - } - defer destFile.Close() - - if _, err := io.Copy(destFile, srcFile); err != nil { - return fmt.Errorf("error copying file contents: %w", err) + for _, src := range srcFiles { + if err := copyFile(src, dest); err != nil { + return err + } } return nil }, } - cmd.Flags().StringVar( - &src, + cmd.Flags().StringSliceVar( + &srcFiles, srcFlag, - "", - "The source file to be copied", + []string{}, + "The source files to be copied", ) cmd.Flags().StringVar( &dest, destFlag, "", - "The destination for the source file to be copied to", + "The destination directory for the source files to be copied to", ) cmd.MarkFlagsRequiredTogether(srcFlag, destFlag) @@ -551,6 +569,26 @@ func createCopyCommand() *cobra.Command { return cmd } +func copyFile(src, dest string) error { + srcFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("error opening source file: %w", err) + } + defer srcFile.Close() + + destFile, err := os.Create(filepath.Join(dest, filepath.Base(src))) + if err != nil { + return fmt.Errorf("error creating destination file: %w", err) + } + defer destFile.Close() + + if _, err := io.Copy(destFile, srcFile); err != nil { + return fmt.Errorf("error copying file contents: %w", err) + } + + return nil +} + func parseFlags(flags *pflag.FlagSet) ([]string, []string) { var flagKeys, flagValues []string diff --git a/cmd/gateway/commands_test.go b/cmd/gateway/commands_test.go index a727cbe341..832c44358e 100644 --- a/cmd/gateway/commands_test.go +++ b/cmd/gateway/commands_test.go @@ -2,6 +2,8 @@ package main import ( "io" + "os" + "path/filepath" "testing" . "github.com/onsi/gomega" @@ -148,9 +150,12 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { "--health-disable", "--leader-election-lock-name=my-lock", "--leader-election-disable=false", - "--usage-report-secret=default/my-secret", - "--usage-report-server-url=https://my-api.com", - "--usage-report-cluster-name=my-cluster", + "--nginx-plus", + "--usage-report-secret=my-secret", + "--usage-report-endpoint=example.com", + "--usage-report-resolver=resolver.com", + "--usage-report-ca-secret=ca-secret", + "--usage-report-client-ssl-secret=client-secret", "--snippets-filters", }, wantErr: false, @@ -318,54 +323,76 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { { name: "usage-report-secret is invalid", args: []string{ - "--usage-report-secret=my-secret", // no namespace + "--usage-report-secret=!@#$", }, - wantErr: true, - expectedErrPrefix: `invalid argument "my-secret" for "--usage-report-secret" flag: invalid format; ` + - "must be NAMESPACE/NAME", + wantErr: true, + expectedErrPrefix: `invalid argument "!@#$" for "--usage-report-secret" flag: invalid format: `, }, { - name: "usage-report-server-url is set to empty string", + name: "usage-report-endpoint is set to empty string", args: []string{ - "--usage-report-server-url=", + "--usage-report-endpoint=", }, wantErr: true, - expectedErrPrefix: `invalid argument "" for "--usage-report-server-url" flag: must be set`, + expectedErrPrefix: `invalid argument "" for "--usage-report-endpoint" flag: must be set`, + }, + { + name: "usage-report-endpoint is an invalid endpoint", + args: []string{ + "--usage-report-endpoint=$*(invalid)", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "$*(invalid)" for "--usage-report-endpoint" flag: ` + + `"$*(invalid)" must be a domain name or IP address with optional port`, }, { - name: "usage-report-server-url is an invalid url", + name: "usage-report-resolver is set to empty string", args: []string{ - "--usage-report-server-url=invalid", + "--usage-report-resolver=", }, wantErr: true, - expectedErrPrefix: `invalid argument "invalid" for "--usage-report-server-url" flag: "invalid" must be a valid URL`, + expectedErrPrefix: `invalid argument "" for "--usage-report-resolver" flag: must be set`, }, { - name: "usage secret and server url not specified together", + name: "usage-report-resolver is an invalid endpoint", args: []string{ - "--gateway-ctlr-name=gateway.nginx.org/nginx-gateway", - "--gatewayclass=nginx", - "--usage-report-server-url=http://example.com", + "--usage-report-resolver=$*(invalid)", }, wantErr: true, - expectedErrPrefix: "if any flags in the group [usage-report-secret usage-report-server-url] " + - "are set they must all be set", + expectedErrPrefix: `invalid argument "$*(invalid)" for "--usage-report-resolver" flag: ` + + `"$*(invalid)" must be a domain name or IP address with optional port`, }, { - name: "usage-report-cluster-name is set to empty string", + name: "usage-report-ca-secret is set to empty string", args: []string{ - "--usage-report-cluster-name=", + "--usage-report-ca-secret=", }, wantErr: true, - expectedErrPrefix: `invalid argument "" for "--usage-report-cluster-name" flag: must be set`, + expectedErrPrefix: `invalid argument "" for "--usage-report-ca-secret" flag: must be set`, }, { - name: "usage-report-cluster-name is invalid", + name: "usage-report-ca-secret is invalid", args: []string{ - "--usage-report-cluster-name=$invalid*(#)", + "--usage-report-ca-secret=!@#$", }, wantErr: true, - expectedErrPrefix: `invalid argument "$invalid*(#)" for "--usage-report-cluster-name" flag: invalid format`, + expectedErrPrefix: `invalid argument "!@#$" for "--usage-report-ca-secret" flag: invalid format: `, + }, + { + name: "usage-report-client-ssl-secret is set to empty string", + args: []string{ + "--usage-report-client-ssl-secret=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--usage-report-client-ssl-secret" flag: must be set`, + }, + { + name: "usage-report-client-ssl-secret is invalid", + args: []string{ + "--usage-report-client-ssl-secret=!@#$", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "!@#$" for "--usage-report-client-ssl-secret" flag: invalid format: `, }, { name: "snippets-filters is not a bool", @@ -492,6 +519,23 @@ func TestCopyCmdFlagValidation(t *testing.T) { } } +func TestCopyFile(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + src, err := os.CreateTemp(os.TempDir(), "testfile") + g.Expect(err).ToNot(HaveOccurred()) + defer os.Remove(src.Name()) + + dest, err := os.MkdirTemp(os.TempDir(), "testdir") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dest) + + g.Expect(copyFile(src.Name(), dest)).To(Succeed()) + _, err = os.Stat(filepath.Join(dest, filepath.Base(src.Name()))) + g.Expect(err).ToNot(HaveOccurred()) +} + func TestParseFlags(t *testing.T) { t.Parallel() g := NewWithT(t) diff --git a/cmd/gateway/validation.go b/cmd/gateway/validation.go index 536fa4432d..7d5d0f3a5d 100644 --- a/cmd/gateway/validation.go +++ b/cmd/gateway/validation.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "net" - "net/url" "regexp" "strconv" "strings" @@ -104,24 +103,6 @@ func validateQualifiedName(name string) error { return nil } -func validateURL(value string) error { - if len(value) == 0 { - return errors.New("must be set") - } - val, err := url.Parse(value) - if err != nil { - return fmt.Errorf("%q must be a valid URL: %w", value, err) - } - if val.Scheme == "" { - return fmt.Errorf("%q must be a valid URL: bad scheme", value) - } - if val.Host == "" { - return fmt.Errorf("%q must be a valid URL: bad host", value) - } - - return nil -} - func validateIP(ip string) error { if ip == "" { return errors.New("IP address must be set") @@ -162,6 +143,47 @@ func validateEndpoint(endpoint string) error { return fmt.Errorf("%q must be in the format :", endpoint) } +func validateEndpointOptionalPort(value string) error { + if len(value) == 0 { + return errors.New("must be set") + } + + // This function assumes a port exists. If it doesn't, ignore those errors. Any errors with the endpoint + // will be caught by further validation. + host, port, err := net.SplitHostPort(value) + if err != nil && + (!strings.Contains(err.Error(), "missing port") && !strings.Contains(err.Error(), "too many colons")) { + return fmt.Errorf("error splitting %q into host and port: %w", value, err) + } + + if port != "" { + portVal, err := strconv.ParseInt(port, 10, 16) + if err != nil { + return fmt.Errorf("port must be a valid number: %w", err) + } + + if portVal < 1 || portVal > 65535 { + return fmt.Errorf("port outside of valid port range [1 - 65535]: %v", port) + } + } + + if host == "" { + host = value + } + + if err := validateIP(host); err == nil { + return nil + } + + if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 { + return nil + } + + // we don't know if the user intended to use a hostname or an IP address, + // so we return a generic error message + return fmt.Errorf("%q must be a domain name or IP address with optional port", value) +} + // validatePort makes sure a given port is inside the valid port range for its usage. func validatePort(port int) error { if port < 1024 || port > 65535 { @@ -183,3 +205,15 @@ func ensureNoPortCollisions(ports ...int) error { return nil } + +// validateSleepArgs ensures that arguments to the sleep command are set. +func validateSleepArgs(srcFiles []string, dest string) error { + if len(srcFiles) == 0 { + return errors.New("source must not be empty") + } + if len(dest) == 0 { + return errors.New("destination must not be empty") + } + + return nil +} diff --git a/cmd/gateway/validation_test.go b/cmd/gateway/validation_test.go index ebeea67e31..28fbfbf3c3 100644 --- a/cmd/gateway/validation_test.go +++ b/cmd/gateway/validation_test.go @@ -327,70 +327,6 @@ func TestValidateQualifiedName(t *testing.T) { } } -func TestValidateURL(t *testing.T) { - t.Parallel() - tests := []struct { - name string - url string - expErr bool - }{ - { - name: "valid", - url: "http://server.com", - expErr: false, - }, - { - name: "valid https", - url: "https://server.com", - expErr: false, - }, - { - name: "valid with port", - url: "http://server.com:8080", - expErr: false, - }, - { - name: "valid with ip address", - url: "http://10.0.0.1", - expErr: false, - }, - { - name: "valid with ip address and port", - url: "http://10.0.0.1:8080", - expErr: false, - }, - { - name: "invalid scheme", - url: "http//server.com", - expErr: true, - }, - { - name: "no scheme", - url: "server.com", - expErr: true, - }, - { - name: "no domain", - url: "http://", - expErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - g := NewWithT(t) - - err := validateURL(tc.url) - if !tc.expErr { - g.Expect(err).ToNot(HaveOccurred()) - } else { - g.Expect(err).To(HaveOccurred()) - } - }) - } -} - func TestValidateIP(t *testing.T) { t.Parallel() tests := []struct { @@ -502,6 +438,75 @@ func TestValidateEndpoint(t *testing.T) { } } +func TestValidateEndpointOptionalPort(t *testing.T) { + t.Parallel() + tests := []struct { + name string + endp string + expErr bool + }{ + { + name: "valid endpoint with hostname", + endp: "localhost:8080", + expErr: false, + }, + { + name: "valid endpoint with IPv4", + endp: "1.2.3.4:8080", + expErr: false, + }, + { + name: "valid endpoint with IPv6", + endp: "[::1]:8080", + expErr: false, + }, + { + name: "valid endpoint with hostname, no port", + endp: "localhost", + expErr: false, + }, + { + name: "valid endpoint with IPv4, no port", + endp: "1.2.3.4", + expErr: false, + }, + { + name: "valid endpoint with IPv6, no port", + endp: "2041:0000:140F::875B:131B", + expErr: false, + }, + { + name: "invalid port - 1", + endp: "localhost:0", + expErr: true, + }, + { + name: "invalid port - 2", + endp: "localhost:65536", + expErr: true, + }, + { + name: "invalid hostname or IP", + endp: "loc@lhost:8080", + expErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + err := validateEndpointOptionalPort(tc.endp) + if !tc.expErr { + g.Expect(err).ToNot(HaveOccurred()) + } else { + g.Expect(err).To(HaveOccurred()) + } + }) + } +} + func TestValidatePort(t *testing.T) { t.Parallel() tests := []struct { @@ -548,3 +553,47 @@ func TestEnsureNoPortCollisions(t *testing.T) { g.Expect(ensureNoPortCollisions(9113, 8081)).To(Succeed()) g.Expect(ensureNoPortCollisions(9113, 9113)).ToNot(Succeed()) } + +func TestValidateSleepArgs(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + dest string + srcFiles []string + expErr bool + }{ + { + name: "valid values", + dest: "/dest/file", + srcFiles: []string{"/src/file"}, + expErr: false, + }, + { + name: "invalid dest", + dest: "", + srcFiles: []string{"/src/file"}, + expErr: true, + }, + { + name: "invalid src", + dest: "/dest/file", + srcFiles: []string{}, + expErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + err := validateSleepArgs(tc.srcFiles, tc.dest) + if !tc.expErr { + g.Expect(err).ToNot(HaveOccurred()) + } else { + g.Expect(err).To(HaveOccurred()) + } + }) + } +} diff --git a/config/tests/static-deployment.yaml b/config/tests/static-deployment.yaml index 8de0432701..c1aac6dfff 100644 --- a/config/tests/static-deployment.yaml +++ b/config/tests/static-deployment.yaml @@ -31,7 +31,7 @@ spec: - --source - /includes/main.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes securityContext: seccompProfile: type: RuntimeDefault @@ -44,7 +44,7 @@ spec: runAsUser: 102 runAsGroup: 1001 volumeMounts: - - name: nginx-includes-configmap + - name: nginx-includes-bootstrap mountPath: /includes - name: nginx-main-includes mountPath: /etc/nginx/main-includes @@ -141,6 +141,8 @@ spec: mountPath: /var/run/nginx - name: nginx-cache mountPath: /var/cache/nginx + - name: nginx-lib + mountPath: /var/lib/nginx - name: nginx-includes mountPath: /etc/nginx/includes terminationGracePeriodSeconds: 30 @@ -162,8 +164,10 @@ spec: emptyDir: {} - name: nginx-cache emptyDir: {} + - name: nginx-lib + emptyDir: {} - name: nginx-includes emptyDir: {} - - name: nginx-includes-configmap + - name: nginx-includes-bootstrap configMap: - name: nginx-includes + name: nginx-includes-bootstrap diff --git a/deploy/aws-nlb/deploy.yaml b/deploy/aws-nlb/deploy.yaml index 91ce6f5f06..53de84437e 100644 --- a/deploy/aws-nlb/deploy.yaml +++ b/deploy/aws-nlb/deploy.yaml @@ -152,7 +152,7 @@ metadata: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: nginx-gateway app.kubernetes.io/version: edge - name: nginx-includes + name: nginx-includes-bootstrap namespace: nginx-gateway --- apiVersion: v1 @@ -301,6 +301,8 @@ spec: name: nginx-run - mountPath: /var/cache/nginx name: nginx-cache + - mountPath: /var/lib/nginx + name: nginx-lib - mountPath: /etc/nginx/includes name: nginx-includes initContainers: @@ -310,7 +312,7 @@ spec: - --source - /includes/main.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: copy-nginx-config @@ -327,7 +329,7 @@ spec: type: RuntimeDefault volumeMounts: - mountPath: /includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes name: nginx-main-includes securityContext: @@ -349,11 +351,13 @@ spec: name: nginx-run - emptyDir: {} name: nginx-cache + - emptyDir: {} + name: nginx-lib - emptyDir: {} name: nginx-includes - configMap: - name: nginx-includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap + name: nginx-includes-bootstrap --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index 1b1c209eef..1b365b557a 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -152,7 +152,7 @@ metadata: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: nginx-gateway app.kubernetes.io/version: edge - name: nginx-includes + name: nginx-includes-bootstrap namespace: nginx-gateway --- apiVersion: v1 @@ -298,6 +298,8 @@ spec: name: nginx-run - mountPath: /var/cache/nginx name: nginx-cache + - mountPath: /var/lib/nginx + name: nginx-lib - mountPath: /etc/nginx/includes name: nginx-includes initContainers: @@ -307,7 +309,7 @@ spec: - --source - /includes/main.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: copy-nginx-config @@ -324,7 +326,7 @@ spec: type: RuntimeDefault volumeMounts: - mountPath: /includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes name: nginx-main-includes nodeSelector: @@ -348,11 +350,13 @@ spec: name: nginx-run - emptyDir: {} name: nginx-cache + - emptyDir: {} + name: nginx-lib - emptyDir: {} name: nginx-includes - configMap: - name: nginx-includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap + name: nginx-includes-bootstrap --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index 0d19731785..1892520591 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -152,7 +152,7 @@ metadata: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: nginx-gateway app.kubernetes.io/version: edge - name: nginx-includes + name: nginx-includes-bootstrap namespace: nginx-gateway --- apiVersion: v1 @@ -298,6 +298,8 @@ spec: name: nginx-run - mountPath: /var/cache/nginx name: nginx-cache + - mountPath: /var/lib/nginx + name: nginx-lib - mountPath: /etc/nginx/includes name: nginx-includes initContainers: @@ -307,7 +309,7 @@ spec: - --source - /includes/main.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: copy-nginx-config @@ -324,7 +326,7 @@ spec: type: RuntimeDefault volumeMounts: - mountPath: /includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes name: nginx-main-includes securityContext: @@ -346,11 +348,13 @@ spec: name: nginx-run - emptyDir: {} name: nginx-cache + - emptyDir: {} + name: nginx-lib - emptyDir: {} name: nginx-includes - configMap: - name: nginx-includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap + name: nginx-includes-bootstrap --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index 9e22545f68..0b155ec02b 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -159,13 +159,17 @@ apiVersion: v1 data: main.conf: | error_log stderr info; + mgmt.conf: | + mgmt { + enforce_initial_report off; + } kind: ConfigMap metadata: labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: nginx-gateway app.kubernetes.io/version: edge - name: nginx-includes + name: nginx-includes-bootstrap namespace: nginx-gateway --- apiVersion: v1 @@ -225,6 +229,7 @@ spec: - --config=nginx-gateway-config - --service=nginx-gateway - --nginx-plus + - --usage-report-secret=nplus-license - --metrics-port=9113 - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election @@ -313,16 +318,23 @@ spec: name: nginx-run - mountPath: /var/cache/nginx name: nginx-cache + - mountPath: /var/lib/nginx + name: nginx-lib - mountPath: /etc/nginx/includes name: nginx-includes + - mountPath: /etc/nginx/license.jwt + name: nginx-plus-license + subPath: license.jwt initContainers: - command: - /usr/bin/gateway - copy - --source - /includes/main.conf + - --source + - /includes/mgmt.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: copy-nginx-config @@ -339,7 +351,7 @@ spec: type: RuntimeDefault volumeMounts: - mountPath: /includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes name: nginx-main-includes securityContext: @@ -361,11 +373,16 @@ spec: name: nginx-run - emptyDir: {} name: nginx-cache + - emptyDir: {} + name: nginx-lib - emptyDir: {} name: nginx-includes - configMap: - name: nginx-includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap + name: nginx-includes-bootstrap + - name: nginx-plus-license + secret: + secretName: nplus-license --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index 6642fe5c36..6580c470ea 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -157,7 +157,7 @@ metadata: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: nginx-gateway app.kubernetes.io/version: edge - name: nginx-includes + name: nginx-includes-bootstrap namespace: nginx-gateway --- apiVersion: v1 @@ -304,6 +304,8 @@ spec: name: nginx-run - mountPath: /var/cache/nginx name: nginx-cache + - mountPath: /var/lib/nginx + name: nginx-lib - mountPath: /etc/nginx/includes name: nginx-includes initContainers: @@ -313,7 +315,7 @@ spec: - --source - /includes/main.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: copy-nginx-config @@ -330,7 +332,7 @@ spec: type: RuntimeDefault volumeMounts: - mountPath: /includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes name: nginx-main-includes securityContext: @@ -352,11 +354,13 @@ spec: name: nginx-run - emptyDir: {} name: nginx-cache + - emptyDir: {} + name: nginx-lib - emptyDir: {} name: nginx-includes - configMap: - name: nginx-includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap + name: nginx-includes-bootstrap --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index 9a9762c662..145c89722a 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -154,13 +154,17 @@ apiVersion: v1 data: main.conf: | error_log stderr info; + mgmt.conf: | + mgmt { + enforce_initial_report off; + } kind: ConfigMap metadata: labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: nginx-gateway app.kubernetes.io/version: edge - name: nginx-includes + name: nginx-includes-bootstrap namespace: nginx-gateway --- apiVersion: v1 @@ -220,11 +224,10 @@ spec: - --config=nginx-gateway-config - --service=nginx-gateway - --nginx-plus + - --usage-report-secret=nplus-license - --metrics-port=9113 - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election - - --usage-report-secret=nginx-gateway/ngf-usage-auth - - --usage-report-server-url=https://my-instance-nim.example.com env: - name: POD_IP valueFrom: @@ -309,16 +312,23 @@ spec: name: nginx-run - mountPath: /var/cache/nginx name: nginx-cache + - mountPath: /var/lib/nginx + name: nginx-lib - mountPath: /etc/nginx/includes name: nginx-includes + - mountPath: /etc/nginx/license.jwt + name: nginx-plus-license + subPath: license.jwt initContainers: - command: - /usr/bin/gateway - copy - --source - /includes/main.conf + - --source + - /includes/mgmt.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: copy-nginx-config @@ -335,7 +345,7 @@ spec: type: RuntimeDefault volumeMounts: - mountPath: /includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes name: nginx-main-includes securityContext: @@ -357,11 +367,16 @@ spec: name: nginx-run - emptyDir: {} name: nginx-cache + - emptyDir: {} + name: nginx-lib - emptyDir: {} name: nginx-includes - configMap: - name: nginx-includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap + name: nginx-includes-bootstrap + - name: nginx-plus-license + secret: + secretName: nplus-license --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index 9c42cccd88..8ba5d32da5 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -152,7 +152,7 @@ metadata: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: nginx-gateway app.kubernetes.io/version: edge - name: nginx-includes + name: nginx-includes-bootstrap namespace: nginx-gateway --- apiVersion: v1 @@ -298,6 +298,8 @@ spec: name: nginx-run - mountPath: /var/cache/nginx name: nginx-cache + - mountPath: /var/lib/nginx + name: nginx-lib - mountPath: /etc/nginx/includes name: nginx-includes initContainers: @@ -307,7 +309,7 @@ spec: - --source - /includes/main.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: copy-nginx-config @@ -324,7 +326,7 @@ spec: type: RuntimeDefault volumeMounts: - mountPath: /includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes name: nginx-main-includes securityContext: @@ -346,11 +348,13 @@ spec: name: nginx-run - emptyDir: {} name: nginx-cache + - emptyDir: {} + name: nginx-lib - emptyDir: {} name: nginx-includes - configMap: - name: nginx-includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap + name: nginx-includes-bootstrap --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index c17d5c2e98..7ed4a4a176 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -160,7 +160,7 @@ metadata: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: nginx-gateway app.kubernetes.io/version: edge - name: nginx-includes + name: nginx-includes-bootstrap namespace: nginx-gateway --- apiVersion: v1 @@ -306,6 +306,8 @@ spec: name: nginx-run - mountPath: /var/cache/nginx name: nginx-cache + - mountPath: /var/lib/nginx + name: nginx-lib - mountPath: /etc/nginx/includes name: nginx-includes initContainers: @@ -315,7 +317,7 @@ spec: - --source - /includes/main.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: copy-nginx-config @@ -332,7 +334,7 @@ spec: type: RuntimeDefault volumeMounts: - mountPath: /includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes name: nginx-main-includes securityContext: @@ -354,11 +356,13 @@ spec: name: nginx-run - emptyDir: {} name: nginx-cache + - emptyDir: {} + name: nginx-lib - emptyDir: {} name: nginx-includes - configMap: - name: nginx-includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap + name: nginx-includes-bootstrap --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -425,3 +429,4 @@ volumes: - emptyDir - secret - configMap +- projected diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index 7cc456b8ee..8bbb1a127a 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -156,13 +156,17 @@ apiVersion: v1 data: main.conf: | error_log stderr info; + mgmt.conf: | + mgmt { + enforce_initial_report off; + } kind: ConfigMap metadata: labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: nginx-gateway app.kubernetes.io/version: edge - name: nginx-includes + name: nginx-includes-bootstrap namespace: nginx-gateway --- apiVersion: v1 @@ -222,6 +226,7 @@ spec: - --config=nginx-gateway-config - --service=nginx-gateway - --nginx-plus + - --usage-report-secret=nplus-license - --metrics-port=9113 - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election @@ -310,16 +315,23 @@ spec: name: nginx-run - mountPath: /var/cache/nginx name: nginx-cache + - mountPath: /var/lib/nginx + name: nginx-lib - mountPath: /etc/nginx/includes name: nginx-includes + - mountPath: /etc/nginx/license.jwt + name: nginx-plus-license + subPath: license.jwt initContainers: - command: - /usr/bin/gateway - copy - --source - /includes/main.conf + - --source + - /includes/mgmt.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: copy-nginx-config @@ -336,7 +348,7 @@ spec: type: RuntimeDefault volumeMounts: - mountPath: /includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes name: nginx-main-includes securityContext: @@ -358,11 +370,16 @@ spec: name: nginx-run - emptyDir: {} name: nginx-cache + - emptyDir: {} + name: nginx-lib - emptyDir: {} name: nginx-includes - configMap: - name: nginx-includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap + name: nginx-includes-bootstrap + - name: nginx-plus-license + secret: + secretName: nplus-license --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index 27d566f983..b623a668fd 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -154,7 +154,7 @@ metadata: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: nginx-gateway app.kubernetes.io/version: edge - name: nginx-includes + name: nginx-includes-bootstrap namespace: nginx-gateway --- apiVersion: v1 @@ -301,6 +301,8 @@ spec: name: nginx-run - mountPath: /var/cache/nginx name: nginx-cache + - mountPath: /var/lib/nginx + name: nginx-lib - mountPath: /etc/nginx/includes name: nginx-includes initContainers: @@ -310,7 +312,7 @@ spec: - --source - /includes/main.conf - --destination - - /etc/nginx/main-includes/main.conf + - /etc/nginx/main-includes image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: copy-nginx-config @@ -327,7 +329,7 @@ spec: type: RuntimeDefault volumeMounts: - mountPath: /includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes name: nginx-main-includes securityContext: @@ -349,11 +351,13 @@ spec: name: nginx-run - emptyDir: {} name: nginx-cache + - emptyDir: {} + name: nginx-lib - emptyDir: {} name: nginx-includes - configMap: - name: nginx-includes - name: nginx-includes-configmap + name: nginx-includes-bootstrap + name: nginx-includes-bootstrap --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index cfe3610351..3c08ad2d7f 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -121,6 +121,7 @@ This will build the docker images `nginx-gateway-fabric:` and `nginx- > Note: You will need a valid NGINX Plus license certificate and key named `nginx-repo.crt` and `nginx-repo.key` in the > root of this repo to build the NGINX Plus image. +> You will also need a valid NGINX Plus JSON Web Token (JWT) to deploy NGF with NGINX Plus. That JWT should be stored in the `license.jwt` file in the root of the `nginx-gateway-fabric/` directory. To build the NGINX Gateway Fabric and NGINX Plus container images from source run the following make command: diff --git a/examples/helm/nginx-plus/values.yaml b/examples/helm/nginx-plus/values.yaml index e702df5e9e..b8b842d16a 100644 --- a/examples/helm/nginx-plus/values.yaml +++ b/examples/helm/nginx-plus/values.yaml @@ -5,9 +5,6 @@ nginx: plus: true image: repository: private-registry.nginx.com/nginx-gateway-fabric/nginx-plus - usage: - secretName: nginx-gateway/ngf-usage-auth - serverURL: https://my-instance-nim.example.com serviceAccount: imagePullSecret: nginx-plus-registry-secret diff --git a/internal/mode/static/config/config.go b/internal/mode/static/config/config.go index 9424deb5c2..dbcbfa6b81 100644 --- a/internal/mode/static/config/config.go +++ b/internal/mode/static/config/config.go @@ -11,8 +11,8 @@ import ( type Config struct { // AtomicLevel is an atomically changeable, dynamic logging level. AtomicLevel zap.AtomicLevel - // UsageReportConfig specifies the NGINX Plus usage reporting config. - UsageReportConfig *UsageReportConfig + // UsageReportConfig specifies the NGINX Plus usage reporting configuration. + UsageReportConfig UsageReportConfig // Version is the running NGF version. Version string // ImageSource is the source of the NGINX Gateway image. @@ -104,14 +104,18 @@ type ProductTelemetryConfig struct { // UsageReportConfig contains the configuration for NGINX Plus usage reporting. type UsageReportConfig struct { - // SecretNsName is the namespaced name of the Secret containing the server credentials. - SecretNsName types.NamespacedName - // ServerURL is the base URL of the reporting server. - ServerURL string - // ClusterDisplayName is the display name of the cluster. Optional. - ClusterDisplayName string - // InsecureSkipVerify controls whether the client verifies the server cert. - InsecureSkipVerify bool + // SecretName is the name of the Secret containing the server credentials. + SecretName string + // ClientSSLSecretName is the name of the Secret containing client certificate/key. + ClientSSLSecretName string + // CASecretName is the name of the Secret containing the CA certificate. + CASecretName string + // Endpoint is the endpoint of the reporting server. + Endpoint string + // Resolver is the nameserver for resolving the Endpoint. + Resolver string + // SkipVerify controls whether the nginx verifies the server certificate. + SkipVerify bool } // Flags contains the NGF command-line flag names and values. diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index 12b0384b5c..637dc378c7 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -29,23 +29,13 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/status" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/telemetry" ) type handlerMetricsCollector interface { ObserveLastEventBatchProcessTime(time.Duration) } -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate -//counterfeiter:generate . secretStorer - -// secretStorer should store the usage Secret that contains the credentials for NGINX Plus usage reporting. -type secretStorer interface { - // Set stores the updated Secret. - Set(*v1.Secret) - // Delete nullifies the Secret value. - Delete() -} - // eventHandlerConfig holds configuration parameters for eventHandlerImpl. type eventHandlerConfig struct { // nginxFileMgr is the file Manager for nginx. @@ -56,22 +46,20 @@ type eventHandlerConfig struct { nginxRuntimeMgr runtime.Manager // statusUpdater updates statuses on Kubernetes resources. statusUpdater frameworkStatus.GroupUpdater - // usageSecret contains the Secret for the NGINX Plus reporting credentials. - usageSecret secretStorer // processor is the state ChangeProcessor. processor state.ChangeProcessor // serviceResolver resolves Services to Endpoints. serviceResolver resolver.ServiceResolver // generator is the nginx config generator. generator ngxConfig.Generator - // k8sClient is a Kubernetes API client + // k8sClient is a Kubernetes API client. k8sClient client.Client + // k8sReader is a Kubernets API reader. + k8sReader client.Reader // logLevelSetter is used to update the logging level. logLevelSetter logLevelSetter // eventRecorder records events for Kubernetes resources. eventRecorder record.EventRecorder - // usageReportConfig contains the configuration for NGINX Plus usage reporting. - usageReportConfig *ngfConfig.UsageReportConfig // nginxConfiguredOnStartChecker sets the health of the Pod to Ready once we've written out our initial config. nginxConfiguredOnStartChecker *nginxConfiguredOnStartChecker // gatewayPodConfig contains information about this Pod. @@ -82,6 +70,8 @@ type eventHandlerConfig struct { gatewayCtlrName string // updateGatewayClassStatus enables updating the status of the GatewayClass resource. updateGatewayClassStatus bool + // plus is whether or not we are running NGINX Plus. + plus bool } const ( @@ -133,9 +123,8 @@ func newEventHandlerImpl(cfg eventHandlerConfig) *eventHandlerImpl { handler.objectFilters = map[filterKey]objectFilter{ // NginxGateway CRD objectFilterKey(&ngfAPI.NginxGateway{}, handler.cfg.controlConfigNSName): { - upsert: handler.nginxGatewayCRDUpsert, - delete: handler.nginxGatewayCRDDelete, - captureChangeInGraph: false, + upsert: handler.nginxGatewayCRDUpsert, + delete: handler.nginxGatewayCRDDelete, }, // NGF-fronting Service objectFilterKey( @@ -151,16 +140,6 @@ func newEventHandlerImpl(cfg eventHandlerConfig) *eventHandlerImpl { }, } - if handler.cfg.usageReportConfig != nil { - // N+ usage reporting Secret - nsName := handler.cfg.usageReportConfig.SecretNsName - handler.objectFilters[objectFilterKey(&v1.Secret{}, nsName)] = objectFilter{ - upsert: handler.nginxPlusUsageSecretUpsert, - delete: handler.nginxPlusUsageSecretDelete, - captureChangeInGraph: true, - } - } - return handler } @@ -194,6 +173,11 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log case state.EndpointsOnlyChange: h.version++ cfg := dataplane.BuildConfiguration(ctx, gr, h.cfg.serviceResolver, h.version) + depCtx, setErr := h.setDeploymentCtx(ctx, logger) + if setErr != nil { + logger.Error(setErr, "error setting deployment context for usage reporting") + } + cfg.DeploymentContext = depCtx h.setLatestConfiguration(&cfg) @@ -205,6 +189,11 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log case state.ClusterStateChange: h.version++ cfg := dataplane.BuildConfiguration(ctx, gr, h.cfg.serviceResolver, h.version) + depCtx, setErr := h.setDeploymentCtx(ctx, logger) + if setErr != nil { + logger.Error(setErr, "error setting deployment context for usage reporting") + } + cfg.DeploymentContext = depCtx h.setLatestConfiguration(&cfg) @@ -511,6 +500,47 @@ func getGatewayAddresses( return gwAddresses, nil } +// setDeploymentCtx sets the deployment context metadata for nginx plus reporting. +func (h *eventHandlerImpl) setDeploymentCtx( + ctx context.Context, + logger logr.Logger, +) (dataplane.DeploymentContext, error) { + if !h.cfg.plus { + return dataplane.DeploymentContext{}, nil + } + + podNSName := types.NamespacedName{ + Name: h.cfg.gatewayPodConfig.Name, + Namespace: h.cfg.gatewayPodConfig.Namespace, + } + + clusterInfo, err := telemetry.CollectClusterInformation(ctx, h.cfg.k8sReader) + if err != nil { + return dataplane.DeploymentContext{}, fmt.Errorf("error getting cluster information: %w", err) + } + + var installationID string + // InstallationID is not required by the usage API, so if we can't get it, don't return an error + replicaSet, err := telemetry.GetPodReplicaSet(ctx, h.cfg.k8sReader, podNSName) + if err != nil { + logger.Error(err, "failed to get NGF installationID") + } else { + installationID, err = telemetry.GetDeploymentID(replicaSet) + if err != nil { + logger.Error(err, "failed to get NGF installationID") + } + } + + depCtx := dataplane.DeploymentContext{ + Integration: "ngf", + ClusterID: clusterInfo.ClusterID, + ClusterNodeCount: clusterInfo.NodeCount, + InstallationID: installationID, + } + + return depCtx, nil +} + // GetLatestConfiguration gets the latest configuration. func (h *eventHandlerImpl) GetLatestConfiguration() *dataplane.Configuration { h.lock.Lock() @@ -609,20 +639,3 @@ func (h *eventHandlerImpl) nginxGatewayServiceDelete( ) h.cfg.statusUpdater.UpdateGroup(ctx, groupGateways, gatewayStatuses...) } - -func (h *eventHandlerImpl) nginxPlusUsageSecretUpsert(_ context.Context, _ logr.Logger, obj client.Object) { - secret, ok := obj.(*v1.Secret) - if !ok { - panic(fmt.Errorf("obj type mismatch: got %T, expected %T", obj, &v1.Secret{})) - } - - h.cfg.usageSecret.Set(secret) -} - -func (h *eventHandlerImpl) nginxPlusUsageSecretDelete( - _ context.Context, - _ logr.Logger, - _ types.NamespacedName, -) { - h.cfg.usageSecret.Delete() -} diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index a912168a2f..01fc3c0c54 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" discoveryV1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,7 +33,6 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/statefakes" - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/staticfakes" ) var _ = Describe("eventHandler", func() { @@ -404,95 +404,6 @@ var _ = Describe("eventHandler", func() { }) }) - When("receiving usage Secret updates", func() { - var fakeSecretStore *staticfakes.FakeSecretStorer - var usageSecret *v1.Secret - - BeforeEach(func() { - fakeSecretStore = &staticfakes.FakeSecretStorer{} - usageSecret = &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "usage", - Namespace: "nginx-gateway", - }, - } - }) - - It("should not set the N+ usage secret if not initialized", func() { - handler.cfg.usageSecret = fakeSecretStore - - e := &events.UpsertEvent{ - Resource: usageSecret, - } - batch := []interface{}{e} - - handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) - Expect(fakeSecretStore.SetCallCount()).To(Equal(0)) - Expect(fakeProcessor.CaptureUpsertChangeCallCount()).To(Equal(1)) - }) - - Context("usage secret is initialized", func() { - var usageSecretHandler *eventHandlerImpl - BeforeEach(func() { - usageCfg := &config.UsageReportConfig{ - SecretNsName: client.ObjectKeyFromObject(usageSecret), - } - usageSecretHandler = newEventHandlerImpl(eventHandlerConfig{ - k8sClient: fake.NewFakeClient(), - processor: fakeProcessor, - nginxConfiguredOnStartChecker: newNginxConfiguredOnStartChecker(), - controlConfigNSName: types.NamespacedName{Namespace: namespace, Name: configName}, - usageReportConfig: usageCfg, - usageSecret: fakeSecretStore, - gatewayPodConfig: config.GatewayPodConfig{ - ServiceName: "nginx-gateway", - Namespace: "nginx-gateway", - }, - metricsCollector: collectors.NewControllerNoopCollector(), - }) - }) - - It("should not set the N+ usage secret if processing a normal secret", func() { - e := &events.UpsertEvent{ - Resource: &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "not-usage", - Namespace: "nginx-gateway", - }, - }, - } - batch := []interface{}{e} - - usageSecretHandler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) - Expect(fakeSecretStore.SetCallCount()).To(Equal(0)) - Expect(fakeProcessor.CaptureUpsertChangeCallCount()).To(Equal(1)) - }) - - It("should set the N+ usage secret when upserted", func() { - e := &events.UpsertEvent{ - Resource: usageSecret, - } - batch := []interface{}{e} - - usageSecretHandler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) - Expect(fakeSecretStore.SetCallCount()).To(Equal(1)) - Expect(fakeProcessor.CaptureUpsertChangeCallCount()).To(Equal(1)) - }) - - It("should remove the N+ usage secret when deleted", func() { - e := &events.DeleteEvent{ - Type: &v1.Secret{}, - NamespacedName: client.ObjectKeyFromObject(usageSecret), - } - batch := []interface{}{e} - - usageSecretHandler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) - Expect(fakeSecretStore.DeleteCallCount()).To(Equal(1)) - Expect(fakeProcessor.CaptureDeleteChangeCallCount()).To(Equal(1)) - }) - }) - }) - When("receiving an EndpointsOnlyChange update", func() { e := &events.UpsertEvent{Resource: &discoveryV1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -786,3 +697,133 @@ var _ = Describe("getGatewayAddresses", func() { Expect(addrs[1].Value).To(Equal("myhost")) }) }) + +var _ = Describe("setDeploymentCtx", func() { + When("nginx plus is false", func() { + It("doesn't set the deployment context", func() { + handler := eventHandlerImpl{} + + depCtx, err := handler.setDeploymentCtx(context.Background(), ctlrZap.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(depCtx).To(Equal(dataplane.DeploymentContext{})) + }) + }) + + When("nginx plus is true", func() { + var ( + clusterID = "test-uid" + ngfPod = &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "replicaset1", + }, + }, + }, + } + + ngfReplicaSet = &appsv1.ReplicaSet{ + Spec: appsv1.ReplicaSetSpec{ + Replicas: helpers.GetPointer[int32](1), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "replicaset1", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Deployment", + Name: "Deployment1", + UID: "test-uid-replicaSet", + }, + }, + }, + } + + kubeNamespace = &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: metav1.NamespaceSystem, + UID: "test-uid", + }, + } + + nodeList = &v1.NodeList{ + Items: []v1.Node{{}}, + } + ) + + It("sets the deployment context", func() { + handler := newEventHandlerImpl(eventHandlerConfig{ + plus: true, + k8sReader: fake.NewFakeClient(ngfPod, ngfReplicaSet, kubeNamespace, nodeList), + gatewayPodConfig: config.GatewayPodConfig{ + Name: ngfPod.Name, + }, + }) + + expCtx := dataplane.DeploymentContext{ + Integration: "ngf", + ClusterID: clusterID, + InstallationID: "test-uid-replicaSet", + ClusterNodeCount: 1, + } + + depCtx, err := handler.setDeploymentCtx(context.Background(), ctlrZap.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(depCtx).To(Equal(expCtx)) + }) + + It("returns an error if cluster info isn't found", func() { + handler := newEventHandlerImpl(eventHandlerConfig{ + plus: true, + k8sReader: fake.NewFakeClient(), + }) + + _, err := handler.setDeploymentCtx(context.Background(), ctlrZap.New()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("error getting cluster information")) + }) + + It("sets the deployment context when the replicaset isn't found", func() { + handler := newEventHandlerImpl(eventHandlerConfig{ + plus: true, + k8sReader: fake.NewFakeClient(ngfPod, kubeNamespace, nodeList), + gatewayPodConfig: config.GatewayPodConfig{ + Name: ngfPod.Name, + }, + }) + + expCtx := dataplane.DeploymentContext{ + Integration: "ngf", + ClusterID: clusterID, + ClusterNodeCount: 1, + } + + depCtx, err := handler.setDeploymentCtx(context.Background(), ctlrZap.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(depCtx).To(Equal(expCtx)) + }) + + It("sets the deployment context when the replicaset doesn't have a uid", func() { + ngfReplicaSet.ObjectMeta.OwnerReferences[0].UID = "" + + handler := newEventHandlerImpl(eventHandlerConfig{ + plus: true, + k8sReader: fake.NewFakeClient(ngfPod, ngfReplicaSet, kubeNamespace, nodeList), + gatewayPodConfig: config.GatewayPodConfig{ + Name: ngfPod.Name, + }, + }) + + expCtx := dataplane.DeploymentContext{ + Integration: "ngf", + ClusterID: clusterID, + ClusterNodeCount: 1, + } + + depCtx, err := handler.setDeploymentCtx(context.Background(), ctlrZap.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(depCtx).To(Equal(expCtx)) + }) + }) +}) diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index f5fb6dafbd..4c80fc4260 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -2,7 +2,6 @@ package static import ( "context" - "errors" "fmt" "os" "time" @@ -57,15 +56,20 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/file" ngxruntime "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/runtime" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/validation" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/telemetry" - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/usage" ) const ( // clusterTimeout is a timeout for connections to the Kubernetes API. clusterTimeout = 10 * time.Second + // the following are the names of data fields within NGINX Plus related Secrets. + plusLicenseField = "license.jwt" + plusCAField = "ca.crt" + plusClientCertField = "tls.crt" + plusClientKeyField = "tls.key" ) var scheme = runtime.NewScheme() @@ -107,8 +111,7 @@ func StartManager(cfg config.Config) error { Namespace: cfg.GatewayPodConfig.Namespace, Name: cfg.ConfigName, } - err = registerControllers(ctx, cfg, mgr, recorder, logLevelSetter, eventCh, controlConfigNSName) - if err != nil { + if err := registerControllers(ctx, cfg, mgr, recorder, logLevelSetter, eventCh, controlConfigNSName); err != nil { return err } @@ -123,6 +126,11 @@ func StartManager(cfg config.Config) error { genericValidator := ngxvalidation.GenericValidator{} policyManager := createPolicyManager(mustExtractGVK, genericValidator) + plusSecrets, err := createPlusSecretMetadata(cfg, mgr.GetAPIReader()) + if err != nil { + return err + } + processor := state.NewChangeProcessorImpl(state.ChangeProcessorConfig{ GatewayCtlrName: cfg.GatewayCtlrName, GatewayClassName: cfg.GatewayClassName, @@ -135,8 +143,19 @@ func StartManager(cfg config.Config) error { EventRecorder: recorder, MustExtractGVK: mustExtractGVK, ProtectedPorts: protectedPorts, + PlusSecrets: plusSecrets, }) + // Clear the configuration folders to ensure that no files are left over in case the control plane was restarted + // (this assumes the folders are in a shared volume). + removedPaths, err := file.ClearFolders(file.NewStdLibOSFileManager(), ngxcfg.ConfigFolders) + for _, path := range removedPaths { + cfg.Logger.Info("removed configuration file", "path", path) + } + if err != nil { + return fmt.Errorf("cannot clear NGINX configuration folders: %w", err) + } + processHandler := ngxruntime.NewProcessHandlerImpl(os.ReadFile, os.Stat) // Ensure NGINX is running before registering metrics & starting the manager. @@ -152,25 +171,6 @@ func StartManager(cfg config.Config) error { ) var ngxPlusClient ngxruntime.NginxPlusClient - var usageSecret *usage.Secret - - if cfg.Plus { - if cfg.UsageReportConfig != nil { - usageSecret = usage.NewUsageSecret() - reporter, err := createUsageReporterJob(mgr.GetAPIReader(), cfg, usageSecret, nginxChecker.getReadyCh()) - if err != nil { - return fmt.Errorf("error creating usage reporter job") - } - - if err = mgr.Add(reporter); err != nil { - return fmt.Errorf("cannot register usage reporter: %w", err) - } - } else { - if err = mgr.Add(createUsageWarningJob(cfg, nginxChecker.getReadyCh())); err != nil { - return fmt.Errorf("cannot register usage warning job: %w", err) - } - } - } if cfg.MetricsConfig.Enabled { constLabels := map[string]string{"class": cfg.GatewayClassName} @@ -216,10 +216,15 @@ func StartManager(cfg config.Config) error { eventHandler := newEventHandlerImpl(eventHandlerConfig{ k8sClient: mgr.GetClient(), + k8sReader: mgr.GetAPIReader(), processor: processor, serviceResolver: resolver.NewServiceResolverImpl(mgr.GetClient()), - generator: ngxcfg.NewGeneratorImpl(cfg.Plus), - logLevelSetter: logLevelSetter, + generator: ngxcfg.NewGeneratorImpl( + cfg.Plus, + &cfg.UsageReportConfig, + cfg.Logger.WithName("generator"), + ), + logLevelSetter: logLevelSetter, nginxFileMgr: file.NewManagerImpl( cfg.Logger.WithName("nginxFileManager"), file.NewStdLibOSFileManager(), @@ -237,10 +242,9 @@ func StartManager(cfg config.Config) error { controlConfigNSName: controlConfigNSName, gatewayPodConfig: cfg.GatewayPodConfig, metricsCollector: handlerCollector, - usageReportConfig: cfg.UsageReportConfig, - usageSecret: usageSecret, gatewayCtlrName: cfg.GatewayCtlrName, updateGatewayClassStatus: cfg.UpdateGatewayClassStatus, + plus: cfg.Plus, }) objects, objectLists := prepareFirstEventBatchPreparerArgs(cfg) @@ -563,6 +567,91 @@ func registerControllers( return nil } +func createPlusSecretMetadata( + cfg config.Config, + reader client.Reader, +) (map[types.NamespacedName][]graph.PlusSecretFile, error) { + plusSecrets := make(map[types.NamespacedName][]graph.PlusSecretFile) + if cfg.Plus { + jwtSecretName := types.NamespacedName{ + Namespace: cfg.GatewayPodConfig.Namespace, + Name: cfg.UsageReportConfig.SecretName, + } + + if err := validateSecret(reader, jwtSecretName, plusLicenseField); err != nil { + return nil, err + } + + jwtSecretCfg := graph.PlusSecretFile{ + FieldName: plusLicenseField, + Type: graph.PlusReportJWTToken, + } + + plusSecrets[jwtSecretName] = []graph.PlusSecretFile{jwtSecretCfg} + + if cfg.UsageReportConfig.CASecretName != "" { + caSecretName := types.NamespacedName{ + Namespace: cfg.GatewayPodConfig.Namespace, + Name: cfg.UsageReportConfig.CASecretName, + } + + if err := validateSecret(reader, caSecretName, plusCAField); err != nil { + return nil, err + } + + caSecretCfg := graph.PlusSecretFile{ + FieldName: plusCAField, + Type: graph.PlusReportCACertificate, + } + + plusSecrets[caSecretName] = []graph.PlusSecretFile{caSecretCfg} + } + + if cfg.UsageReportConfig.ClientSSLSecretName != "" { + clientSSLSecretName := types.NamespacedName{ + Namespace: cfg.GatewayPodConfig.Namespace, + Name: cfg.UsageReportConfig.ClientSSLSecretName, + } + + if err := validateSecret(reader, clientSSLSecretName, plusClientCertField, plusClientKeyField); err != nil { + return nil, err + } + + clientSSLCertCfg := graph.PlusSecretFile{ + FieldName: plusClientCertField, + Type: graph.PlusReportClientSSLCertificate, + } + + clientSSLKeyCfg := graph.PlusSecretFile{ + FieldName: plusClientKeyField, + Type: graph.PlusReportClientSSLKey, + } + + plusSecrets[clientSSLSecretName] = []graph.PlusSecretFile{clientSSLCertCfg, clientSSLKeyCfg} + } + } + + return plusSecrets, nil +} + +func validateSecret(reader client.Reader, nsName types.NamespacedName, fields ...string) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var secret apiv1.Secret + if err := reader.Get(ctx, nsName, &secret); err != nil { + return fmt.Errorf("error getting %q Secret: %w", nsName.Name, err) + } + + for _, field := range fields { + if _, ok := secret.Data[field]; !ok { + return fmt.Errorf("secret %q does not have expected field %q", nsName.Name, field) + } + } + + return nil +} + // 10 min jitter is enough per telemetry destination recommendation // For the default period of 24 hours, jitter will be 10min /(24*60)min = 0.0069. const telemetryJitterFactor = 10.0 / (24 * 60) // added jitter is bound by jitterFactor * period @@ -614,52 +703,6 @@ func createTelemetryJob( }, nil } -func createUsageReporterJob( - k8sClient client.Reader, - cfg config.Config, - usageSecret *usage.Secret, - readyCh <-chan struct{}, -) (*runnables.Leader, error) { - logger := cfg.Logger.WithName("usageReporter") - reporter, err := usage.NewNIMReporter( - usageSecret, - cfg.UsageReportConfig.ServerURL, - cfg.UsageReportConfig.InsecureSkipVerify, - ) - if err != nil { - return nil, err - } - - return &runnables.Leader{ - Runnable: runnables.NewCronJob(runnables.CronJobConfig{ - Worker: usage.CreateUsageJobWorker(logger, k8sClient, reporter, cfg), - Logger: logger, - Period: cfg.ProductTelemetryConfig.ReportPeriod, - JitterFactor: telemetryJitterFactor, - ReadyCh: readyCh, - }), - }, nil -} - -func createUsageWarningJob(cfg config.Config, readyCh <-chan struct{}) *runnables.LeaderOrNonLeader { - logger := cfg.Logger.WithName("usageReporter") - worker := func(_ context.Context) { - logger.Error( - errors.New("usage reporting not enabled"), - "Usage reporting must be enabled when using NGINX Plus; redeploy with usage reporting enabled", - ) - } - - return &runnables.LeaderOrNonLeader{ - Runnable: runnables.NewCronJob(runnables.CronJobConfig{ - Worker: worker, - Logger: logger, - Period: 1 * time.Hour, - ReadyCh: readyCh, - }), - } -} - func prepareFirstEventBatchPreparerArgs(cfg config.Config) ([]client.Object, []client.ObjectList) { objects := []client.Object{ &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: cfg.GatewayClassName}}, diff --git a/internal/mode/static/manager_test.go b/internal/mode/static/manager_test.go index 2954f46d19..5a61a75854 100644 --- a/internal/mode/static/manager_test.go +++ b/internal/mode/static/manager_test.go @@ -8,9 +8,11 @@ import ( discoveryV1 "k8s.io/api/discovery/v1" apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -19,6 +21,7 @@ import ( ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" ) func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) { @@ -254,3 +257,294 @@ func TestGetMetricsOptions(t *testing.T) { }) } } + +func TestCreatePlusSecretMetadata(t *testing.T) { + t.Parallel() + + jwtSecret := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ngf", + Name: "nplus-license", + }, + Data: map[string][]byte{ + plusLicenseField: []byte("data"), + }, + } + + jwtSecretWrongField := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ngf", + Name: "nplus-license", + }, + Data: map[string][]byte{ + "wrong": []byte("data"), + }, + } + + caSecret := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ngf", + Name: "ca", + }, + Data: map[string][]byte{ + plusCAField: []byte("data"), + }, + } + + caSecretWrongField := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ngf", + Name: "ca", + }, + Data: map[string][]byte{ + "wrong": []byte("data"), + }, + } + + clientSecret := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ngf", + Name: "client", + }, + Data: map[string][]byte{ + plusClientCertField: []byte("data"), + plusClientKeyField: []byte("data"), + }, + } + + clientSecretWrongCert := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ngf", + Name: "client", + }, + Data: map[string][]byte{ + "wrong": []byte("data"), + plusClientKeyField: []byte("data"), + }, + } + + clientSecretWrongKey := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ngf", + Name: "client", + }, + Data: map[string][]byte{ + plusClientCertField: []byte("data"), + "wrong": []byte("data"), + }, + } + + tests := []struct { + expSecrets map[types.NamespacedName][]graph.PlusSecretFile + name string + secrets []runtime.Object + cfg config.Config + expErr bool + }{ + { + name: "plus not enabled", + cfg: config.Config{ + Plus: false, + }, + expSecrets: map[types.NamespacedName][]graph.PlusSecretFile{}, + }, + { + name: "only JWT token specified", + secrets: []runtime.Object{jwtSecret}, + cfg: config.Config{ + Plus: true, + GatewayPodConfig: config.GatewayPodConfig{Namespace: jwtSecret.Namespace}, + UsageReportConfig: config.UsageReportConfig{ + SecretName: jwtSecret.Name, + }, + }, + expSecrets: map[types.NamespacedName][]graph.PlusSecretFile{ + {Name: jwtSecret.Name, Namespace: jwtSecret.Namespace}: { + { + FieldName: plusLicenseField, + Type: graph.PlusReportJWTToken, + }, + }, + }, + }, + { + name: "JWT and CA specified", + secrets: []runtime.Object{jwtSecret, caSecret}, + cfg: config.Config{ + Plus: true, + GatewayPodConfig: config.GatewayPodConfig{Namespace: jwtSecret.Namespace}, + UsageReportConfig: config.UsageReportConfig{ + SecretName: jwtSecret.Name, + CASecretName: caSecret.Name, + }, + }, + expSecrets: map[types.NamespacedName][]graph.PlusSecretFile{ + {Name: jwtSecret.Name, Namespace: jwtSecret.Namespace}: { + { + FieldName: plusLicenseField, + Type: graph.PlusReportJWTToken, + }, + }, + {Name: caSecret.Name, Namespace: jwtSecret.Namespace}: { + { + FieldName: plusCAField, + Type: graph.PlusReportCACertificate, + }, + }, + }, + }, + { + name: "all Secrets specified", + secrets: []runtime.Object{jwtSecret, caSecret, clientSecret}, + cfg: config.Config{ + Plus: true, + GatewayPodConfig: config.GatewayPodConfig{Namespace: jwtSecret.Namespace}, + UsageReportConfig: config.UsageReportConfig{ + SecretName: jwtSecret.Name, + CASecretName: caSecret.Name, + ClientSSLSecretName: clientSecret.Name, + }, + }, + expSecrets: map[types.NamespacedName][]graph.PlusSecretFile{ + {Name: jwtSecret.Name, Namespace: jwtSecret.Namespace}: { + { + FieldName: plusLicenseField, + Type: graph.PlusReportJWTToken, + }, + }, + {Name: caSecret.Name, Namespace: jwtSecret.Namespace}: { + { + FieldName: plusCAField, + Type: graph.PlusReportCACertificate, + }, + }, + {Name: clientSecret.Name, Namespace: jwtSecret.Namespace}: { + { + FieldName: plusClientCertField, + Type: graph.PlusReportClientSSLCertificate, + }, + { + FieldName: plusClientKeyField, + Type: graph.PlusReportClientSSLKey, + }, + }, + }, + }, + { + name: "JWT Secret doesn't exist", + cfg: config.Config{ + Plus: true, + GatewayPodConfig: config.GatewayPodConfig{Namespace: jwtSecret.Namespace}, + UsageReportConfig: config.UsageReportConfig{ + SecretName: jwtSecret.Name, + }, + }, + expSecrets: nil, + expErr: true, + }, + { + name: "JWT Secret doesn't have correct field", + secrets: []runtime.Object{jwtSecretWrongField}, + cfg: config.Config{ + Plus: true, + GatewayPodConfig: config.GatewayPodConfig{Namespace: jwtSecret.Namespace}, + UsageReportConfig: config.UsageReportConfig{ + SecretName: jwtSecret.Name, + }, + }, + expSecrets: nil, + expErr: true, + }, + { + name: "CA Secret doesn't exist", + secrets: []runtime.Object{jwtSecret}, + cfg: config.Config{ + Plus: true, + GatewayPodConfig: config.GatewayPodConfig{Namespace: jwtSecret.Namespace}, + UsageReportConfig: config.UsageReportConfig{ + SecretName: jwtSecret.Name, + CASecretName: caSecret.Name, + }, + }, + expSecrets: nil, + expErr: true, + }, + { + name: "CA Secret doesn't have correct field", + secrets: []runtime.Object{jwtSecretWrongField, caSecretWrongField}, + cfg: config.Config{ + Plus: true, + GatewayPodConfig: config.GatewayPodConfig{Namespace: jwtSecret.Namespace}, + UsageReportConfig: config.UsageReportConfig{ + SecretName: jwtSecret.Name, + CASecretName: caSecret.Name, + }, + }, + expSecrets: nil, + expErr: true, + }, + { + name: "Client Secret doesn't exist", + secrets: []runtime.Object{jwtSecret, caSecret}, + cfg: config.Config{ + Plus: true, + GatewayPodConfig: config.GatewayPodConfig{Namespace: jwtSecret.Namespace}, + UsageReportConfig: config.UsageReportConfig{ + SecretName: jwtSecret.Name, + CASecretName: caSecret.Name, + ClientSSLSecretName: clientSecret.Name, + }, + }, + expSecrets: nil, + expErr: true, + }, + { + name: "Client Secret doesn't have correct cert", + secrets: []runtime.Object{jwtSecret, caSecret, clientSecretWrongCert}, + cfg: config.Config{ + Plus: true, + GatewayPodConfig: config.GatewayPodConfig{Namespace: jwtSecret.Namespace}, + UsageReportConfig: config.UsageReportConfig{ + SecretName: jwtSecret.Name, + CASecretName: caSecret.Name, + ClientSSLSecretName: clientSecret.Name, + }, + }, + expSecrets: nil, + expErr: true, + }, + { + name: "Client Secret doesn't have correct key", + secrets: []runtime.Object{jwtSecret, caSecret, clientSecretWrongKey}, + cfg: config.Config{ + Plus: true, + GatewayPodConfig: config.GatewayPodConfig{Namespace: jwtSecret.Namespace}, + UsageReportConfig: config.UsageReportConfig{ + SecretName: jwtSecret.Name, + CASecretName: caSecret.Name, + ClientSSLSecretName: clientSecret.Name, + }, + }, + expSecrets: nil, + expErr: true, + }, + } + + for _, test := range tests { + fakeClient := fake.NewFakeClient(test.secrets...) + + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + plusSecrets, err := createPlusSecretMetadata(test.cfg, fakeClient) + if test.expErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + + g.Expect(plusSecrets).To(Equal(test.expSecrets)) + }) + } +} diff --git a/internal/mode/static/nginx/conf/nginx-plus.conf b/internal/mode/static/nginx/conf/nginx-plus.conf index 77526ac8ad..17ac6de4e3 100644 --- a/internal/mode/static/nginx/conf/nginx-plus.conf +++ b/internal/mode/static/nginx/conf/nginx-plus.conf @@ -66,7 +66,3 @@ stream { access_log /dev/stdout stream-main; include /etc/nginx/stream-conf.d/*.conf; } - -mgmt { - usage_report interval=0s; -} diff --git a/internal/mode/static/nginx/config/generator.go b/internal/mode/static/nginx/config/generator.go index 31c9d60ee2..9792eb4d9a 100644 --- a/internal/mode/static/nginx/config/generator.go +++ b/internal/mode/static/nginx/config/generator.go @@ -3,6 +3,9 @@ package config import ( "path/filepath" + "github.com/go-logr/logr" + + ngfConfig "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/clientsettings" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/observability" @@ -13,6 +16,7 @@ import ( //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate //counterfeiter:generate . Generator +// Volumes here also need to be added to our crossplane ephemeral test container. const ( // configFolder is the folder where NGINX configuration files are stored. configFolder = "/etc/nginx" @@ -47,6 +51,9 @@ const ( // mainIncludesConfigFile is the path to the file containing NGINX configuration in the main context. mainIncludesConfigFile = mainIncludesFolder + "/main.conf" + + // mgmtIncludesFile is the path to the file containing the NGINX Plus mgmt config. + mgmtIncludesFile = mainIncludesFolder + "/mgmt.conf" ) // ConfigFolders is a list of folders where NGINX configuration files are stored. @@ -62,17 +69,27 @@ type Generator interface { // GeneratorImpl is an implementation of Generator. // -// It generates files to be written to the ConfigFolders locations, which must exist and available for writing. +// It generates files to be written to the folders above, which must exist and available for writing. // // It also expects that the main NGINX configuration file nginx.conf is located in configFolder and nginx.conf -// includes (https://nginx.org/en/docs/ngx_core_module.html#include) the files from httpFolder. +// includes (https://nginx.org/en/docs/ngx_core_module.html#include) the files from other folders. type GeneratorImpl struct { - plus bool + usageReportConfig *ngfConfig.UsageReportConfig + logger logr.Logger + plus bool } // NewGeneratorImpl creates a new GeneratorImpl. -func NewGeneratorImpl(plus bool) GeneratorImpl { - return GeneratorImpl{plus: plus} +func NewGeneratorImpl( + plus bool, + usageReportConfig *ngfConfig.UsageReportConfig, + logger logr.Logger, +) GeneratorImpl { + return GeneratorImpl{ + plus: plus, + usageReportConfig: usageReportConfig, + logger: logger, + } } type executeResult struct { @@ -88,7 +105,7 @@ type executeFunc func(configuration dataplane.Configuration) []executeResult // In case of invalid configuration, NGINX will fail to reload or could be configured with malicious configuration. // To validate, use the validators from the validation package. func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File { - files := make([]file.File, 0, len(conf.SSLKeyPairs)+1 /* http config */) + files := make([]file.File, 0) for id, pair := range conf.SSLKeyPairs { files = append(files, generatePEM(id, pair.Cert, pair.Key)) @@ -121,7 +138,12 @@ func (g GeneratorImpl) executeConfigTemplates( } } - files := make([]file.File, 0, len(fileBytes)) + var mgmtFiles []file.File + if g.plus { + mgmtFiles = g.generateMgmtFiles(conf) + } + + files := make([]file.File, 0, len(fileBytes)+len(mgmtFiles)) for fp, bytes := range fileBytes { files = append(files, file.File{ Path: fp, @@ -129,6 +151,7 @@ func (g GeneratorImpl) executeConfigTemplates( Type: file.TypeRegular, }) } + files = append(files, mgmtFiles...) return files } diff --git a/internal/mode/static/nginx/config/generator_test.go b/internal/mode/static/nginx/config/generator_test.go index ad698aa38a..f343e3dc42 100644 --- a/internal/mode/static/nginx/config/generator_test.go +++ b/internal/mode/static/nginx/config/generator_test.go @@ -7,10 +7,13 @@ import ( . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/types" + ctlrZap "sigs.k8s.io/controller-runtime/pkg/log/zap" + ngfConfig "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/file" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" ) @@ -116,15 +119,31 @@ func TestGenerate(t *testing.T) { Contents: "main 2 contents", }, }, + DeploymentContext: dataplane.DeploymentContext{ + Integration: "ngf", + ClusterID: "test-uid", + InstallationID: "test-uid-replicaSet", + ClusterNodeCount: 1, + }, + AuxiliarySecrets: map[graph.SecretFileType][]byte{ + graph.PlusReportJWTToken: []byte("license"), + graph.PlusReportCACertificate: []byte("ca"), + graph.PlusReportClientSSLCertificate: []byte("cert"), + graph.PlusReportClientSSLKey: []byte("key"), + }, } g := NewWithT(t) - var plus bool - generator := config.NewGeneratorImpl(plus) + plus := true + generator := config.NewGeneratorImpl( + plus, + &ngfConfig.UsageReportConfig{Endpoint: "test-endpoint"}, + ctlrZap.New(), + ) files := generator.Generate(conf) - g.Expect(files).To(HaveLen(11)) + g.Expect(files).To(HaveLen(17)) arrange := func(i, j int) bool { return files[i].Path < files[j].Path } @@ -139,7 +158,13 @@ func TestGenerate(t *testing.T) { /etc/nginx/includes/http_snippet2.conf /etc/nginx/includes/main_snippet1.conf /etc/nginx/includes/main_snippet2.conf + /etc/nginx/main-includes/deployment_ctx.json /etc/nginx/main-includes/main.conf + /etc/nginx/main-includes/mgmt.conf + /etc/nginx/secrets/license.jwt + /etc/nginx/secrets/mgmt-ca.crt + /etc/nginx/secrets/mgmt-tls.crt + /etc/nginx/secrets/mgmt-tls.key /etc/nginx/secrets/test-certbundle.crt /etc/nginx/secrets/test-keypair.pem /etc/nginx/stream-conf.d/stream.conf @@ -182,25 +207,53 @@ func TestGenerate(t *testing.T) { g.Expect(files[5].Path).To(Equal("/etc/nginx/includes/main_snippet1.conf")) g.Expect(files[6].Path).To(Equal("/etc/nginx/includes/main_snippet2.conf")) - g.Expect(files[7].Path).To(Equal("/etc/nginx/main-includes/main.conf")) - mainConfStr := string(files[7].Content) + g.Expect(files[7].Path).To(Equal("/etc/nginx/main-includes/deployment_ctx.json")) + deploymentCtx := string(files[7].Content) + g.Expect(deploymentCtx).To(ContainSubstring("\"integration\":\"ngf\"")) + g.Expect(deploymentCtx).To(ContainSubstring("\"cluster_id\":\"test-uid\"")) + g.Expect(deploymentCtx).To(ContainSubstring("\"installation_id\":\"test-uid-replicaSet\"")) + g.Expect(deploymentCtx).To(ContainSubstring("\"cluster_node_count\":1")) + + g.Expect(files[8].Path).To(Equal("/etc/nginx/main-includes/main.conf")) + mainConfStr := string(files[8].Content) g.Expect(mainConfStr).To(ContainSubstring("load_module modules/ngx_otel_module.so;")) g.Expect(mainConfStr).To(ContainSubstring("include /etc/nginx/includes/main_snippet1.conf;")) g.Expect(mainConfStr).To(ContainSubstring("include /etc/nginx/includes/main_snippet2.conf;")) - g.Expect(files[8].Path).To(Equal("/etc/nginx/secrets/test-certbundle.crt")) - certBundle := string(files[8].Content) + g.Expect(files[9].Path).To(Equal("/etc/nginx/main-includes/mgmt.conf")) + mgmtConf := string(files[9].Content) + g.Expect(mgmtConf).To(ContainSubstring("usage_report endpoint=test-endpoint")) + g.Expect(mgmtConf).To(ContainSubstring("license_token /etc/nginx/secrets/license.jwt")) + g.Expect(mgmtConf).To(ContainSubstring("deployment_context /etc/nginx/main-includes/deployment_ctx.json")) + g.Expect(mgmtConf).To(ContainSubstring("ssl_trusted_certificate /etc/nginx/secrets/mgmt-ca.crt")) + g.Expect(mgmtConf).To(ContainSubstring("ssl_certificate /etc/nginx/secrets/mgmt-tls.crt")) + g.Expect(mgmtConf).To(ContainSubstring("ssl_certificate_key /etc/nginx/secrets/mgmt-tls.key")) + + g.Expect(files[10].Path).To(Equal("/etc/nginx/secrets/license.jwt")) + g.Expect(string(files[10].Content)).To(Equal("license")) + + g.Expect(files[11].Path).To(Equal("/etc/nginx/secrets/mgmt-ca.crt")) + g.Expect(string(files[11].Content)).To(Equal("ca")) + + g.Expect(files[12].Path).To(Equal("/etc/nginx/secrets/mgmt-tls.crt")) + g.Expect(string(files[12].Content)).To(Equal("cert")) + + g.Expect(files[13].Path).To(Equal("/etc/nginx/secrets/mgmt-tls.key")) + g.Expect(string(files[13].Content)).To(Equal("key")) + + g.Expect(files[14].Path).To(Equal("/etc/nginx/secrets/test-certbundle.crt")) + certBundle := string(files[14].Content) g.Expect(certBundle).To(Equal("test-cert")) - g.Expect(files[9]).To(Equal(file.File{ + g.Expect(files[15]).To(Equal(file.File{ Type: file.TypeSecret, Path: "/etc/nginx/secrets/test-keypair.pem", Content: []byte("test-cert\ntest-key"), })) - g.Expect(files[10].Path).To(Equal("/etc/nginx/stream-conf.d/stream.conf")) - g.Expect(files[10].Type).To(Equal(file.TypeRegular)) - streamCfg := string(files[10].Content) + g.Expect(files[16].Path).To(Equal("/etc/nginx/stream-conf.d/stream.conf")) + g.Expect(files[16].Type).To(Equal(file.TypeRegular)) + streamCfg := string(files[16].Content) g.Expect(streamCfg).To(ContainSubstring("listen unix:/var/run/nginx/app.example.com-443.sock")) g.Expect(streamCfg).To(ContainSubstring("listen 443")) g.Expect(streamCfg).To(ContainSubstring("app.example.com unix:/var/run/nginx/app.example.com-443.sock")) diff --git a/internal/mode/static/nginx/config/main_config.go b/internal/mode/static/nginx/config/main_config.go index e9a089fbc8..bec292a86f 100644 --- a/internal/mode/static/nginx/config/main_config.go +++ b/internal/mode/static/nginx/config/main_config.go @@ -1,14 +1,20 @@ package config import ( + "encoding/json" gotemplate "text/template" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/shared" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/file" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" ) -var mainConfigTemplate = gotemplate.Must(gotemplate.New("main").Parse(mainConfigTemplateText)) +var ( + mainConfigTemplate = gotemplate.Must(gotemplate.New("main").Parse(mainConfigTemplateText)) + mgmtConfigTemplate = gotemplate.Must(gotemplate.New("mgmt").Parse(mgmtConfigTemplateText)) +) type mainConfig struct { Includes []shared.Include @@ -32,3 +38,93 @@ func executeMainConfig(conf dataplane.Configuration) []executeResult { return results } + +type mgmtConf struct { + Endpoint string + Resolver string + LicenseTokenFile string + DeploymentCtxFile string + CACertFile string + ClientSSLCertFile string + ClientSSLKeyFile string + SkipVerify bool +} + +// generateMgmtFiles generates the NGINX Plus configuration file for the mgmt block. As part of this, +// it writes the secret and deployment context files that are referenced in the mgmt block. +func (g GeneratorImpl) generateMgmtFiles(conf dataplane.Configuration) []file.File { + if !g.plus { + return nil + } + + tokenContent, ok := conf.AuxiliarySecrets[graph.PlusReportJWTToken] + if !ok { + panic("nginx plus token not set in expected map") + } + + tokenFile := file.File{ + Content: tokenContent, + Path: secretsFolder + "/license.jwt", + Type: file.TypeSecret, + } + files := []file.File{tokenFile} + + cfg := mgmtConf{ + Endpoint: g.usageReportConfig.Endpoint, + Resolver: g.usageReportConfig.Resolver, + LicenseTokenFile: tokenFile.Path, + SkipVerify: g.usageReportConfig.SkipVerify, + } + + if content, ok := conf.AuxiliarySecrets[graph.PlusReportCACertificate]; ok { + caFile := file.File{ + Content: content, + Path: secretsFolder + "/mgmt-ca.crt", + Type: file.TypeSecret, + } + cfg.CACertFile = caFile.Path + files = append(files, caFile) + } + + if content, ok := conf.AuxiliarySecrets[graph.PlusReportClientSSLCertificate]; ok { + certFile := file.File{ + Content: content, + Path: secretsFolder + "/mgmt-tls.crt", + Type: file.TypeSecret, + } + cfg.ClientSSLCertFile = certFile.Path + files = append(files, certFile) + } + + if content, ok := conf.AuxiliarySecrets[graph.PlusReportClientSSLKey]; ok { + keyFile := file.File{ + Content: content, + Path: secretsFolder + "/mgmt-tls.key", + Type: file.TypeSecret, + } + cfg.ClientSSLKeyFile = keyFile.Path + files = append(files, keyFile) + } + + depCtx, err := json.Marshal(conf.DeploymentContext) + if err != nil { + g.logger.Error(err, "error building deployment context for mgmt block") + } else { + deploymentCtxFile := file.File{ + Content: depCtx, + Path: mainIncludesFolder + "/deployment_ctx.json", + Type: file.TypeRegular, + } + + cfg.DeploymentCtxFile = deploymentCtxFile.Path + files = append(files, deploymentCtxFile) + } + + mgmtBlockFile := file.File{ + Content: helpers.MustExecuteTemplate(mgmtConfigTemplate, cfg), + Path: mgmtIncludesFile, + Type: file.TypeRegular, + } + + return append(files, mgmtBlockFile) +} diff --git a/internal/mode/static/nginx/config/main_config_template.go b/internal/mode/static/nginx/config/main_config_template.go index f25e0b8d5e..895494f7b2 100644 --- a/internal/mode/static/nginx/config/main_config_template.go +++ b/internal/mode/static/nginx/config/main_config_template.go @@ -11,3 +11,28 @@ error_log stderr {{ .Conf.Logging.ErrorLevel }}; include {{ $i.Name }}; {{ end -}} ` + +const mgmtConfigTemplateText = ` +mgmt { + {{- if .Endpoint }} + usage_report endpoint={{ .Endpoint }}; + {{- end }} + {{- if .Resolver }} + resolver {{ .Resolver }}; + {{- end }} + license_token {{ .LicenseTokenFile }}; + {{- if .DeploymentCtxFile }} + deployment_context {{ .DeploymentCtxFile }}; + {{- end }} + {{- if .SkipVerify }} + ssl_verify off; + {{- end }} + {{- if .CACertFile }} + ssl_trusted_certificate {{ .CACertFile }}; + {{- end }} + {{- if and .ClientSSLCertFile .ClientSSLKeyFile }} + ssl_certificate {{ .ClientSSLCertFile }}; + ssl_certificate_key {{ .ClientSSLKeyFile }}; + {{- end }} +} +` diff --git a/internal/mode/static/nginx/config/main_config_test.go b/internal/mode/static/nginx/config/main_config_test.go index 3f9a50caca..6a5773542f 100644 --- a/internal/mode/static/nginx/config/main_config_test.go +++ b/internal/mode/static/nginx/config/main_config_test.go @@ -125,3 +125,25 @@ func TestExecuteMainConfig_Snippets(t *testing.T) { g.Expect(res[3].dest).To(Equal(mainIncludesConfigFile)) } + +func TestGenerateMgmtFiles_NoPlus(t *testing.T) { + t.Parallel() + + gen := GeneratorImpl{} + files := gen.generateMgmtFiles(dataplane.Configuration{}) + + g := NewWithT(t) + g.Expect(files).To(BeNil()) +} + +func TestGenerateMgmtFiles_Panic(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + gen := GeneratorImpl{plus: true} + + // panics if JWT token is not set in the AuxiliarySecrets map + g.Expect(func() { + gen.generateMgmtFiles(dataplane.Configuration{}) + }).To(Panic()) +} diff --git a/internal/mode/static/nginx/file/folders.go b/internal/mode/static/nginx/file/folders.go index d3a3eebbd5..a7b296e1d1 100644 --- a/internal/mode/static/nginx/file/folders.go +++ b/internal/mode/static/nginx/file/folders.go @@ -19,6 +19,12 @@ type ClearFoldersOSFileManager interface { Remove(name string) error } +// These files are needed on startup, so skip deleting them. +const ( + mainConf = "/etc/nginx/main-includes/main.conf" + mgmtConf = "/etc/nginx/main-includes/mgmt.conf" +) + // ClearFolders removes all files in the given folders and returns the removed files' full paths. func ClearFolders(fileMgr ClearFoldersOSFileManager, paths []string) (removedFiles []string, e error) { for _, path := range paths { @@ -29,6 +35,11 @@ func ClearFolders(fileMgr ClearFoldersOSFileManager, paths []string) (removedFil for _, entry := range entries { entryPath := filepath.Join(path, entry.Name()) + + if entryPath == mainConf || entryPath == mgmtConf { + continue + } + if err := fileMgr.Remove(entryPath); err != nil { return removedFiles, fmt.Errorf("failed to remove %q: %w", entryPath, err) } diff --git a/internal/mode/static/state/change_processor.go b/internal/mode/static/state/change_processor.go index 9da779c7da..cb35491209 100644 --- a/internal/mode/static/state/change_processor.go +++ b/internal/mode/static/state/change_processor.go @@ -70,6 +70,8 @@ type ChangeProcessorConfig struct { MustExtractGVK kinds.MustExtractGVK // ProtectedPorts are the ports that may not be configured by a listener with a descriptive name of the ports. ProtectedPorts graph.ProtectedPorts + // PlusSecrets is a list of secret files used for NGINX Plus reporting (JWT, client SSL, CA). + PlusSecrets map[types.NamespacedName][]graph.PlusSecretFile // Logger is the logger for this Change Processor. Logger logr.Logger // GatewayCtlrName is the name of the Gateway controller. @@ -275,6 +277,7 @@ func (c *ChangeProcessorImpl) Process() (ChangeType, *graph.Graph) { c.clusterState, c.cfg.GatewayCtlrName, c.cfg.GatewayClassName, + c.cfg.PlusSecrets, c.cfg.Validators, c.cfg.ProtectedPorts, ) diff --git a/internal/mode/static/state/dataplane/configuration.go b/internal/mode/static/state/dataplane/configuration.go index 6b2ac1b2dc..f533b67a1b 100644 --- a/internal/mode/static/state/dataplane/configuration.go +++ b/internal/mode/static/state/dataplane/configuration.go @@ -56,6 +56,7 @@ func BuildConfiguration( BaseHTTPConfig: baseHTTPConfig, Logging: buildLogging(g), MainSnippets: buildSnippetsForContext(g.SnippetsFilters, ngfAPI.NginxContextMain), + AuxiliarySecrets: buildAuxiliarySecrets(g.PlusSecrets), } return config @@ -956,9 +957,24 @@ func buildLogging(g *graph.Graph) Logging { return logSettings } +func buildAuxiliarySecrets( + secrets map[types.NamespacedName][]graph.PlusSecretFile, +) map[graph.SecretFileType][]byte { + auxSecrets := make(map[graph.SecretFileType][]byte) + + for _, secretFiles := range secrets { + for _, file := range secretFiles { + auxSecrets[file.Type] = file.Content + } + } + + return auxSecrets +} + func GetDefaultConfiguration(g *graph.Graph, configVersion int) Configuration { return Configuration{ - Version: configVersion, - Logging: buildLogging(g), + Version: configVersion, + Logging: buildLogging(g), + AuxiliarySecrets: buildAuxiliarySecrets(g.PlusSecrets), } } diff --git a/internal/mode/static/state/dataplane/configuration_test.go b/internal/mode/static/state/dataplane/configuration_test.go index f3869cbeb2..7bed89e5d2 100644 --- a/internal/mode/static/state/dataplane/configuration_test.go +++ b/internal/mode/static/state/dataplane/configuration_test.go @@ -4282,3 +4282,42 @@ func TestBuildSnippetForContext(t *testing.T) { }) } } + +func TestBuildAuxiliarySecrets(t *testing.T) { + t.Parallel() + + secrets := map[types.NamespacedName][]graph.PlusSecretFile{ + {Name: "license", Namespace: "ngf"}: { + { + Type: graph.PlusReportJWTToken, + Content: []byte("license"), + }, + }, + {Name: "ca", Namespace: "ngf"}: { + { + Type: graph.PlusReportCACertificate, + Content: []byte("ca"), + }, + }, + {Name: "client", Namespace: "ngf"}: { + { + Type: graph.PlusReportClientSSLCertificate, + Content: []byte("cert"), + }, + { + Type: graph.PlusReportClientSSLKey, + Content: []byte("key"), + }, + }, + } + expSecrets := map[graph.SecretFileType][]byte{ + graph.PlusReportJWTToken: []byte("license"), + graph.PlusReportCACertificate: []byte("ca"), + graph.PlusReportClientSSLCertificate: []byte("cert"), + graph.PlusReportClientSSLKey: []byte("key"), + } + + g := NewWithT(t) + + g.Expect(buildAuxiliarySecrets(secrets)).To(Equal(expSecrets)) +} diff --git a/internal/mode/static/state/dataplane/types.go b/internal/mode/static/state/dataplane/types.go index b096d517ee..274897c007 100644 --- a/internal/mode/static/state/dataplane/types.go +++ b/internal/mode/static/state/dataplane/types.go @@ -7,6 +7,7 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" ) @@ -34,6 +35,11 @@ type Configuration struct { TLSPassthroughServers []Layer4VirtualServer // Upstreams holds all unique http Upstreams. Upstreams []Upstream + // DeploymentContext contains metadata about NGF and the cluster. + DeploymentContext DeploymentContext + // AuxiliarySecrets contains additional secret data, like certificates/keys/tokens that are not related to + // Gateway API resources. + AuxiliarySecrets map[graph.SecretFileType][]byte // StreamUpstreams holds all unique stream Upstreams StreamUpstreams []Upstream // BackendGroups holds all unique BackendGroups. @@ -389,3 +395,16 @@ type Logging struct { // ErrorLevel defines the error log level. ErrorLevel string } + +// DeploymentContext contains metadata about NGF and the cluster. +// This is JSON marshaled into a file created by the generator, hence the json tags. +type DeploymentContext struct { + // Integration is "ngf". + Integration string `json:"integration"` + // ClusterID is the ID of the kube-system namespace. + ClusterID string `json:"cluster_id"` + // InstallationID is the ID of the NGF deployment. + InstallationID string `json:"installation_id"` + // ClusterNodeCount is the count of nodes in the cluster. + ClusterNodeCount int `json:"cluster_node_count"` +} diff --git a/internal/mode/static/state/graph/graph.go b/internal/mode/static/state/graph/graph.go index df39c59423..73b2ecac68 100644 --- a/internal/mode/static/state/graph/graph.go +++ b/internal/mode/static/state/graph/graph.go @@ -1,6 +1,8 @@ package graph import ( + "fmt" + v1 "k8s.io/api/core/v1" discoveryV1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -80,6 +82,8 @@ type Graph struct { GlobalSettings *policies.GlobalSettings // SnippetsFilters holds all the SnippetsFilters. SnippetsFilters map[types.NamespacedName]*SnippetsFilter + // PlusSecrets holds the secrets related to NGINX Plus licensing. + PlusSecrets map[types.NamespacedName][]PlusSecretFile } // ProtectedPorts are the ports that may not be configured by a listener with a descriptive name of each port. @@ -89,8 +93,10 @@ type ProtectedPorts map[int32]string func (g *Graph) IsReferenced(resourceType ngftypes.ObjectType, nsname types.NamespacedName) bool { switch obj := resourceType.(type) { case *v1.Secret: + // Check if secret is a Gateway-referenced Secret, or if it's a Secret used for NGINX Plus reporting. _, exists := g.ReferencedSecrets[nsname] - return exists + _, plusSecretExists := g.PlusSecrets[nsname] + return exists || plusSecretExists case *v1.ConfigMap: _, exists := g.ReferencedCaCertConfigMaps[nsname] return exists @@ -181,6 +187,7 @@ func BuildGraph( state ClusterState, controllerName string, gcName string, + plusSecrets map[types.NamespacedName][]PlusSecretFile, validators validation.Validators, protectedPorts ProtectedPorts, ) *Graph { @@ -253,6 +260,8 @@ func BuildGraph( globalSettings, ) + setPlusSecretContent(state.Secrets, plusSecrets) + g := &Graph{ GatewayClass: gc, Gateway: gw, @@ -269,6 +278,7 @@ func BuildGraph( NGFPolicies: processedPolicies, GlobalSettings: globalSettings, SnippetsFilters: processedSnippetsFilters, + PlusSecrets: plusSecrets, } g.attachPolicies(controllerName) @@ -293,3 +303,49 @@ func gatewayExists( return exists } + +// SecretFileType describes the type of Secret file used for NGINX Plus. +type SecretFileType int + +const ( + // PlusReportJWTToken is the file for the NGINX Plus JWT Token. + PlusReportJWTToken SecretFileType = iota + // PlusReportCACertificate is the file for the NGINX Instance Manager CA certificate. + PlusReportCACertificate + // PlusReportClientSSLCertificate is the file for the NGINX Instance Manager client certificate. + PlusReportClientSSLCertificate + // PlusReportClientSSLKey is the file for the NGINX Instance Manager client key. + PlusReportClientSSLKey +) + +// PlusSecretFile specifies the type and content of an NGINX Plus Secret file. +// A user provides the names of the various Secrets on startup, and we store this info in a map to cross-reference with +// the actual Secrets that exist in k8s. +type PlusSecretFile struct { + // FieldName is the field name within the Secret that holds the data for this file. + FieldName string + // Content is the content of this file. + Content []byte + // Type is the type of Secret file. + Type SecretFileType +} + +// setPlusSecretContent finds the k8s Secret object associated with a PlusSecretFile object, and sets its contents. +func setPlusSecretContent( + clusterSecrets map[types.NamespacedName]*v1.Secret, + plusSecrets map[types.NamespacedName][]PlusSecretFile, +) { + for name, plusSecretFiles := range plusSecrets { + if secret, ok := clusterSecrets[name]; ok { + for idx, file := range plusSecretFiles { + content, ok := secret.Data[file.FieldName] + if !ok { + panic(fmt.Errorf("NGINX Plus Secret did not have expected field %q", file.FieldName)) + } + + file.Content = content + plusSecrets[name][idx] = file + } + } + } +} diff --git a/internal/mode/static/state/graph/graph_test.go b/internal/mode/static/state/graph/graph_test.go index 1fbc26ef7c..627e308bc0 100644 --- a/internal/mode/static/state/graph/graph_test.go +++ b/internal/mode/static/state/graph/graph_test.go @@ -33,10 +33,9 @@ func TestBuildGraph(t *testing.T) { const ( gcName = "my-class" controllerName = "my.controller" + testNS = "test" ) - testNs := "test" - protectedPorts := ProtectedPorts{ 9113: "MetricsPort", 8081: "HealthPort", @@ -351,6 +350,16 @@ func TestBuildGraph(t *testing.T) { Type: v1.SecretTypeTLS, } + plusSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ngf", + Name: "plus-secret", + }, + Data: map[string][]byte{ + "license.jwt": []byte("license"), + }, + } + ns := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: testNs, @@ -644,7 +653,8 @@ func TestBuildGraph(t *testing.T) { client.ObjectKeyFromObject(grToServiceNsRefGrant): grToServiceNsRefGrant, }, Secrets: map[types.NamespacedName]*v1.Secret{ - client.ObjectKeyFromObject(secret): secret, + client.ObjectKeyFromObject(secret): secret, + client.ObjectKeyFromObject(plusSecret): plusSecret, }, BackendTLSPolicies: map[types.NamespacedName]*v1alpha3.BackendTLSPolicy{ client.ObjectKeyFromObject(btp.Source): btp.Source, @@ -913,6 +923,15 @@ func TestBuildGraph(t *testing.T) { client.ObjectKeyFromObject(unreferencedSnippetsFilter): processedUnrefSnippetsFilter, client.ObjectKeyFromObject(referencedSnippetsFilter): processedRefSnippetsFilter, }, + PlusSecrets: map[types.NamespacedName][]PlusSecretFile{ + client.ObjectKeyFromObject(plusSecret): { + { + Type: PlusReportJWTToken, + Content: []byte("license"), + FieldName: "license.jwt", + }, + }, + }, } } @@ -968,6 +987,14 @@ func TestBuildGraph(t *testing.T) { test.store, controllerName, gcName, + map[types.NamespacedName][]PlusSecretFile{ + client.ObjectKeyFromObject(plusSecret): { + { + Type: PlusReportJWTToken, + FieldName: "license.jwt", + }, + }, + }, validation.Validators{ HTTPFieldsValidator: &validationfakes.FakeHTTPFieldsValidator{}, GenericValidator: &validationfakes.FakeGenericValidator{}, @@ -1000,6 +1027,12 @@ func TestIsReferenced(t *testing.T) { Name: "secret", }, } + plusSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ngf", + Name: "plus-secret", + }, + } nsInGraph := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -1170,6 +1203,18 @@ func TestIsReferenced(t *testing.T) { graph: graph, expected: true, }, + { + name: "NGINX Plus JWT Secret", + resource: plusSecret, + graph: &Graph{ + PlusSecrets: map[types.NamespacedName][]PlusSecretFile{ + client.ObjectKeyFromObject(plusSecret): { + {Type: PlusReportJWTToken}, + }, + }, + }, + expected: true, + }, { name: "Secret not in ReferencedSecrets with same Namespace and different Name is not referenced", resource: sameNamespaceDifferentNameSecret, diff --git a/internal/mode/static/staticfakes/fake_secret_storer.go b/internal/mode/static/staticfakes/fake_secret_storer.go deleted file mode 100644 index e7bf3e70b5..0000000000 --- a/internal/mode/static/staticfakes/fake_secret_storer.go +++ /dev/null @@ -1,104 +0,0 @@ -// Code generated by counterfeiter. DO NOT EDIT. -package staticfakes - -import ( - "sync" - - v1 "k8s.io/api/core/v1" -) - -type FakeSecretStorer struct { - DeleteStub func() - deleteMutex sync.RWMutex - deleteArgsForCall []struct { - } - SetStub func(*v1.Secret) - setMutex sync.RWMutex - setArgsForCall []struct { - arg1 *v1.Secret - } - invocations map[string][][]interface{} - invocationsMutex sync.RWMutex -} - -func (fake *FakeSecretStorer) Delete() { - fake.deleteMutex.Lock() - fake.deleteArgsForCall = append(fake.deleteArgsForCall, struct { - }{}) - stub := fake.DeleteStub - fake.recordInvocation("Delete", []interface{}{}) - fake.deleteMutex.Unlock() - if stub != nil { - fake.DeleteStub() - } -} - -func (fake *FakeSecretStorer) DeleteCallCount() int { - fake.deleteMutex.RLock() - defer fake.deleteMutex.RUnlock() - return len(fake.deleteArgsForCall) -} - -func (fake *FakeSecretStorer) DeleteCalls(stub func()) { - fake.deleteMutex.Lock() - defer fake.deleteMutex.Unlock() - fake.DeleteStub = stub -} - -func (fake *FakeSecretStorer) Set(arg1 *v1.Secret) { - fake.setMutex.Lock() - fake.setArgsForCall = append(fake.setArgsForCall, struct { - arg1 *v1.Secret - }{arg1}) - stub := fake.SetStub - fake.recordInvocation("Set", []interface{}{arg1}) - fake.setMutex.Unlock() - if stub != nil { - fake.SetStub(arg1) - } -} - -func (fake *FakeSecretStorer) SetCallCount() int { - fake.setMutex.RLock() - defer fake.setMutex.RUnlock() - return len(fake.setArgsForCall) -} - -func (fake *FakeSecretStorer) SetCalls(stub func(*v1.Secret)) { - fake.setMutex.Lock() - defer fake.setMutex.Unlock() - fake.SetStub = stub -} - -func (fake *FakeSecretStorer) SetArgsForCall(i int) *v1.Secret { - fake.setMutex.RLock() - defer fake.setMutex.RUnlock() - argsForCall := fake.setArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeSecretStorer) Invocations() map[string][][]interface{} { - fake.invocationsMutex.RLock() - defer fake.invocationsMutex.RUnlock() - fake.deleteMutex.RLock() - defer fake.deleteMutex.RUnlock() - fake.setMutex.RLock() - defer fake.setMutex.RUnlock() - copiedInvocations := map[string][][]interface{}{} - for key, value := range fake.invocations { - copiedInvocations[key] = value - } - return copiedInvocations -} - -func (fake *FakeSecretStorer) recordInvocation(key string, args []interface{}) { - fake.invocationsMutex.Lock() - defer fake.invocationsMutex.Unlock() - if fake.invocations == nil { - fake.invocations = map[string][][]interface{}{} - } - if fake.invocations[key] == nil { - fake.invocations[key] = [][]interface{}{} - } - fake.invocations[key] = append(fake.invocations[key], args) -} diff --git a/internal/mode/static/telemetry/collector.go b/internal/mode/static/telemetry/collector.go index a9defb5872..0e1eaf06bb 100644 --- a/internal/mode/static/telemetry/collector.go +++ b/internal/mode/static/telemetry/collector.go @@ -138,7 +138,7 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) { return Data{}, errors.New("failed to collect telemetry data: latest graph cannot be nil") } - clusterInfo, err := collectClusterInformation(ctx, c.cfg.K8sClientReader) + clusterInfo, err := CollectClusterInformation(ctx, c.cfg.K8sClientReader) if err != nil { return Data{}, fmt.Errorf("failed to collect cluster information: %w", err) } @@ -148,9 +148,9 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) { return Data{}, fmt.Errorf("failed to collect NGF resource counts: %w", err) } - replicaSet, err := getPodReplicaSet(ctx, c.cfg.K8sClientReader, c.cfg.PodNSName) + replicaSet, err := GetPodReplicaSet(ctx, c.cfg.K8sClientReader, c.cfg.PodNSName) if err != nil { - return Data{}, fmt.Errorf("failed to get replica set for pod %s: %w", c.cfg.PodNSName, err) + return Data{}, fmt.Errorf("failed to get replica set for pod %v: %w", c.cfg.PodNSName, err) } replicaCount, err := getReplicas(replicaSet) @@ -158,7 +158,7 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) { return Data{}, fmt.Errorf("failed to collect NGF replica count: %w", err) } - deploymentID, err := getDeploymentID(replicaSet) + deploymentID, err := GetDeploymentID(replicaSet) if err != nil { return Data{}, fmt.Errorf("failed to get NGF deploymentID: %w", err) } @@ -280,7 +280,8 @@ func computeRouteCount( } } -func getPodReplicaSet( +// GetPodReplicaSet returns the replicaset for the provided Pod. +func GetPodReplicaSet( ctx context.Context, k8sClient client.Reader, podNSName types.NamespacedName, @@ -323,7 +324,8 @@ func getReplicas(replicaSet *appsv1.ReplicaSet) (int, error) { return int(*replicaSet.Spec.Replicas), nil } -func getDeploymentID(replicaSet *appsv1.ReplicaSet) (string, error) { +// GetDeploymentID gets the deployment ID of the provided ReplicaSet. +func GetDeploymentID(replicaSet *appsv1.ReplicaSet) (string, error) { replicaOwnerRefs := replicaSet.GetOwnerReferences() if len(replicaOwnerRefs) != 1 { return "", fmt.Errorf("expected one owner reference of the NGF ReplicaSet, got %d", len(replicaOwnerRefs)) @@ -340,8 +342,8 @@ func getDeploymentID(replicaSet *appsv1.ReplicaSet) (string, error) { return string(replicaOwnerRefs[0].UID), nil } -// CollectClusterID gets the UID of the kube-system namespace. -func CollectClusterID(ctx context.Context, k8sClient client.Reader) (string, error) { +// collectClusterID gets the UID of the kube-system namespace. +func collectClusterID(ctx context.Context, k8sClient client.Reader) (string, error) { key := types.NamespacedName{ Name: metav1.NamespaceSystem, } @@ -352,24 +354,25 @@ func CollectClusterID(ctx context.Context, k8sClient client.Reader) (string, err return string(kubeNamespace.GetUID()), nil } -type clusterInformation struct { +type ClusterInformation struct { Platform string Version string ClusterID string NodeCount int } -func collectClusterInformation(ctx context.Context, k8sClient client.Reader) (clusterInformation, error) { - var clusterInfo clusterInformation +// CollectClusterInformation collects information about the cluster. +func CollectClusterInformation(ctx context.Context, k8sClient client.Reader) (ClusterInformation, error) { + var clusterInfo ClusterInformation var nodes v1.NodeList if err := k8sClient.List(ctx, &nodes); err != nil { - return clusterInformation{}, fmt.Errorf("failed to get NodeList: %w", err) + return ClusterInformation{}, fmt.Errorf("failed to get NodeList: %w", err) } nodeCount := len(nodes.Items) if nodeCount == 0 { - return clusterInformation{}, errors.New("failed to collect cluster information: NodeList length is zero") + return ClusterInformation{}, errors.New("failed to collect cluster information: NodeList length is zero") } clusterInfo.NodeCount = nodeCount @@ -385,15 +388,15 @@ func collectClusterInformation(ctx context.Context, k8sClient client.Reader) (cl var namespaces v1.NamespaceList if err = k8sClient.List(ctx, &namespaces); err != nil { - return clusterInformation{}, fmt.Errorf("failed to collect cluster information: %w", err) + return ClusterInformation{}, fmt.Errorf("failed to collect cluster information: %w", err) } clusterInfo.Platform = getPlatform(node, namespaces) var clusterID string - clusterID, err = CollectClusterID(ctx, k8sClient) + clusterID, err = collectClusterID(ctx, k8sClient) if err != nil { - return clusterInformation{}, fmt.Errorf("failed to collect cluster information: %w", err) + return ClusterInformation{}, fmt.Errorf("failed to collect cluster information: %w", err) } clusterInfo.ClusterID = clusterID diff --git a/internal/mode/static/usage/doc.go b/internal/mode/static/usage/doc.go deleted file mode 100644 index d700daf0f7..0000000000 --- a/internal/mode/static/usage/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package usage is responsible for reporting NGINX Plus usage data. -*/ -package usage diff --git a/internal/mode/static/usage/job_worker.go b/internal/mode/static/usage/job_worker.go deleted file mode 100644 index 43b7b9115e..0000000000 --- a/internal/mode/static/usage/job_worker.go +++ /dev/null @@ -1,95 +0,0 @@ -package usage - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/telemetry" -) - -func CreateUsageJobWorker( - logger logr.Logger, - k8sClient client.Reader, - reporter Reporter, - cfg config.Config, -) func(ctx context.Context) { - return func(ctx context.Context) { - nodeCount, err := CollectNodeCount(ctx, k8sClient) - if err != nil { - logger.Error(err, "Failed to collect node count") - return - } - - podCount, err := GetTotalNGFPodCount(ctx, k8sClient) - if err != nil { - logger.Error(err, "Failed to collect replica count") - return - } - - clusterUID, err := telemetry.CollectClusterID(ctx, k8sClient) - if err != nil { - logger.Error(err, "Failed to collect cluster UID") - return - } - - clusterDetails := ClusterDetails{ - Metadata: Metadata{ - DisplayName: cfg.UsageReportConfig.ClusterDisplayName, - UID: clusterUID, - }, - NodeCount: int64(nodeCount), - PodDetails: PodDetails{ - CurrentPodCounts: CurrentPodsCount{ - DosCount: int64(0), - PodCount: int64(podCount), - WafCount: int64(0), - }, - }, - } - - if err := reporter.Report(ctx, clusterDetails); err != nil { - logger.Error(err, "Failed to report NGINX Plus usage") - } - } -} - -// GetTotalNGFPodCount returns the total count of NGF Pods in the cluster. -// Uses the "app.kubernetes.io/name" label with either value of "nginx-gateway" or "nginx-gateway-fabric". -func GetTotalNGFPodCount(ctx context.Context, k8sClient client.Reader) (int, error) { - labelKey := "app.kubernetes.io/name" - labelVals := map[string]struct{}{ - "nginx-gateway-fabric": {}, - "nginx-gateway": {}, - } - - var rsList appsv1.ReplicaSetList - if err := k8sClient.List(ctx, &rsList, client.HasLabels{labelKey}); err != nil { - return 0, fmt.Errorf("failed to list replicasets: %w", err) - } - - var count int - for _, rs := range rsList.Items { - val := rs.Labels[labelKey] - if _, ok := labelVals[val]; ok && rs.Spec.Replicas != nil { - count += int(*rs.Spec.Replicas) - } - } - - return count, nil -} - -// CollectNodeCount returns the number of nodes in the cluster. -func CollectNodeCount(ctx context.Context, k8sClient client.Reader) (int, error) { - var nodes v1.NodeList - if err := k8sClient.List(ctx, &nodes); err != nil { - return 0, fmt.Errorf("failed to get NodeList: %w", err) - } - - return len(nodes.Items), nil -} diff --git a/internal/mode/static/usage/job_worker_test.go b/internal/mode/static/usage/job_worker_test.go deleted file mode 100644 index ac7676e853..0000000000 --- a/internal/mode/static/usage/job_worker_test.go +++ /dev/null @@ -1,271 +0,0 @@ -package usage_test - -import ( - "context" - "errors" - "testing" - "time" - - . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - "github.com/nginxinc/nginx-gateway-fabric/internal/framework/events/eventsfakes" - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/usage" - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/usage/usagefakes" -) - -func TestCreateUsageJobWorker(t *testing.T) { - t.Parallel() - replicas := int32(1) - ngfReplicaSet := &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway", - Name: "ngf-replicaset", - Labels: map[string]string{ - "app.kubernetes.io/name": "nginx-gateway", - }, - }, - Spec: appsv1.ReplicaSetSpec{ - Replicas: &replicas, - }, - } - - tests := []struct { - name string - listCalls func(_ context.Context, object client.ObjectList, _ ...client.ListOption) error - getCalls func(_ context.Context, _ types.NamespacedName, object client.Object, _ ...client.GetOption) error - expData usage.ClusterDetails - expErr bool - }{ - { - name: "succeeds", - listCalls: func(_ context.Context, object client.ObjectList, _ ...client.ListOption) error { - switch typedList := object.(type) { - case *v1.NodeList: - typedList.Items = append(typedList.Items, v1.Node{}) - return nil - case *appsv1.ReplicaSetList: - typedList.Items = append(typedList.Items, *ngfReplicaSet) - return nil - } - return nil - }, - getCalls: func(_ context.Context, _ types.NamespacedName, object client.Object, _ ...client.GetOption) error { - if typedObject, ok := object.(*v1.Namespace); ok { - typedObject.Name = metav1.NamespaceSystem - typedObject.UID = "1234abcd" - return nil - } - return nil - }, - expData: usage.ClusterDetails{ - Metadata: usage.Metadata{ - UID: "1234abcd", - DisplayName: "my-cluster", - }, - NodeCount: 1, - PodDetails: usage.PodDetails{ - CurrentPodCounts: usage.CurrentPodsCount{ - PodCount: 1, - }, - }, - }, - expErr: false, - }, - { - name: "collect node count fails", - listCalls: func(_ context.Context, object client.ObjectList, _ ...client.ListOption) error { - if _, ok := object.(*v1.NodeList); ok { - return errors.New("failed to collect node list") - } - return nil - }, - getCalls: func(_ context.Context, _ types.NamespacedName, _ client.Object, _ ...client.GetOption) error { - return nil - }, - expData: usage.ClusterDetails{}, - expErr: true, - }, - { - name: "collect replica count fails", - listCalls: func(_ context.Context, object client.ObjectList, _ ...client.ListOption) error { - switch typedList := object.(type) { - case *v1.NodeList: - typedList.Items = append(typedList.Items, v1.Node{}) - return nil - case *appsv1.ReplicaSetList: - return errors.New("failed to collect replica set list") - } - return nil - }, - getCalls: func(_ context.Context, _ types.NamespacedName, _ client.Object, _ ...client.GetOption) error { - return nil - }, - expData: usage.ClusterDetails{}, - expErr: true, - }, - { - name: "collect cluster UID fails", - listCalls: func(_ context.Context, object client.ObjectList, _ ...client.ListOption) error { - switch typedList := object.(type) { - case *v1.NodeList: - typedList.Items = append(typedList.Items, v1.Node{}) - return nil - case *appsv1.ReplicaSetList: - typedList.Items = append(typedList.Items, *ngfReplicaSet) - return nil - } - return nil - }, - getCalls: func(_ context.Context, _ types.NamespacedName, object client.Object, _ ...client.GetOption) error { - if _, ok := object.(*v1.Namespace); ok { - return errors.New("failed to collect namespace") - } - return nil - }, - expData: usage.ClusterDetails{}, - expErr: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - g := NewWithT(t) - - k8sClientReader := &eventsfakes.FakeReader{} - k8sClientReader.ListCalls(test.listCalls) - k8sClientReader.GetCalls(test.getCalls) - - reporter := &usagefakes.FakeReporter{} - - worker := usage.CreateUsageJobWorker( - zap.New(), - k8sClientReader, - reporter, - config.Config{ - GatewayPodConfig: config.GatewayPodConfig{ - Namespace: "nginx-gateway", - Name: "ngf-pod", - }, - UsageReportConfig: &config.UsageReportConfig{ - ClusterDisplayName: "my-cluster", - }, - }, - ) - - timeout := 10 * time.Second - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - worker(ctx) - if test.expErr { - g.Expect(reporter.ReportCallCount()).To(Equal(0)) - } else { - _, data := reporter.ReportArgsForCall(0) - g.Expect(data).To(Equal(test.expData)) - } - }) - } -} - -func TestGetTotalNGFPodCount(t *testing.T) { - t.Parallel() - g := NewWithT(t) - - rs1Replicas := int32(1) - rs1 := &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway", - Name: "ngf-replicaset1", - Labels: map[string]string{ - "app.kubernetes.io/name": "nginx-gateway", - }, - }, - Spec: appsv1.ReplicaSetSpec{ - Replicas: &rs1Replicas, - }, - } - - rs2Replicas := int32(3) - rs2 := &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway-2", - Name: "ngf-replicaset2", - Labels: map[string]string{ - "app.kubernetes.io/name": "nginx-gateway-fabric", - }, - }, - Spec: appsv1.ReplicaSetSpec{ - Replicas: &rs2Replicas, - }, - } - - rs3Replicas := int32(5) - rs3 := &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "default", - Name: "not-ngf", - }, - Spec: appsv1.ReplicaSetSpec{ - Replicas: &rs3Replicas, - }, - } - - rs4 := &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway-3", - Name: "ngf-replicaset-nil", - Labels: map[string]string{ - "app.kubernetes.io/name": "nginx-gateway-fabric", - }, - }, - Spec: appsv1.ReplicaSetSpec{ - Replicas: nil, - }, - } - - k8sClient := fake.NewFakeClient(rs1, rs2, rs3, rs4) - - expCount := 4 - count, err := usage.GetTotalNGFPodCount(context.Background(), k8sClient) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(count).To(Equal(expCount)) -} - -func TestCollectNodeCount(t *testing.T) { - t.Parallel() - g := NewWithT(t) - - node1 := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Spec: v1.NodeSpec{ - ProviderID: "k3s://ip-172-16-0-210", - }, - } - - node2 := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node2", - }, - Spec: v1.NodeSpec{ - ProviderID: "k3s://ip-172-16-0-210", - }, - } - - k8sClient := fake.NewFakeClient(node1, node2) - - expCount := 2 - count, err := usage.CollectNodeCount(context.Background(), k8sClient) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(count).To(Equal(expCount)) -} diff --git a/internal/mode/static/usage/reporter.go b/internal/mode/static/usage/reporter.go deleted file mode 100644 index 43fbcd01d2..0000000000 --- a/internal/mode/static/usage/reporter.go +++ /dev/null @@ -1,145 +0,0 @@ -package usage - -import ( - "bytes" - "context" - "crypto/tls" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" -) - -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate -//counterfeiter:generate . credentialsGetter -//counterfeiter:generate . Reporter - -const apiBasePath = "/api/platform/v1/k8s-usage" - -// ClusterDetails are the k8s usage details for the cluster. -type ClusterDetails struct { - // Metadata contains the cluster metadata. - Metadata Metadata `json:"metadata"` - // PodDetails contain the details about the NGF Pod. - PodDetails PodDetails `json:"pod_details"` - // NodeCount is the count of Nodes in the cluster. - NodeCount int64 `json:"node_count"` -} - -// Metadata contains the cluster metadata. -type Metadata struct { - // DisplayName is a user friendly resource name. It can be used to define - // a longer, and less constrained, name for a resource. - DisplayName string `json:"displayName"` - // UID is the unique identifier for the cluster. - UID string `json:"uid"` -} - -// PodDetails contain the details about the NGF Pod. -type PodDetails struct { - // CurrentPodsCount is the total count of NGF NGINX Plus Pods in the cluster. - CurrentPodCounts CurrentPodsCount `json:"current_pod_counts"` -} - -// CurrentPodsCount is the total count of NGF NGINX Plus Pods in the cluster. -type CurrentPodsCount struct { - // PodCount is the current count of NGF NGINX Plus Pods in the cluster. - PodCount int64 `json:"pod_count"` - // DosCount is the count of Pods with NAP DOS enabled in the cluster. Not applicable for NGF, - // but required as part of the payload. - DosCount int64 `json:"dos_count"` - // WafCount is the count of Pods with NAP WAF enabled in the cluster. Not applicable for NGF, - // but required as part of the payload. - WafCount int64 `json:"waf_count"` -} - -// credentialsGetter get the credentials for NGINX Plus usage reporting. -type credentialsGetter interface { - // GetCredentials returns the base64 encoded username and password from the Secret. - GetCredentials() ([]byte, []byte) -} - -// Reporter reports the NGINX Plus usage info to the provided collector. -type Reporter interface { - Report(context.Context, ClusterDetails) error -} - -// NIMReporter reports the NGINX Plus usage info to NGINX Instance Manager. -type NIMReporter struct { - // credentials contains the credentials for the usage collector. - credentials credentialsGetter - // baseURL is the base server URL of the usage collector. - baseURL *url.URL - // insecureSkipVerify controls whether the client verifies the server cert. Used in testing. - insecureSkipVerify bool -} - -// NewNIMReporter creates a new NIM usage reporter. -func NewNIMReporter( - credentials credentialsGetter, - baseURL string, - insecureSkipVerify bool, -) (*NIMReporter, error) { - serverURL, err := url.Parse(baseURL) - if err != nil { - return nil, fmt.Errorf("error parsing usage server URL: %w", err) - } - - return &NIMReporter{ - credentials: credentials, - baseURL: serverURL, - insecureSkipVerify: insecureSkipVerify, - }, nil -} - -// Report sends a PUT request with the provided data to the API endpoint configured in the Reporter. -// The clusterUID is used as the name in the API path. -func (r *NIMReporter) Report(ctx context.Context, data ClusterDetails) error { - buf, err := json.Marshal(data) - if err != nil { - return fmt.Errorf("error marshaling usage data: %w", err) - } - - queryURL := r.baseURL.JoinPath(apiBasePath, data.Metadata.UID) - bodyReader := bytes.NewReader(buf) - - req, err := http.NewRequestWithContext(ctx, http.MethodPut, queryURL.String(), bodyReader) - if err != nil { - return fmt.Errorf("error creating usage API HTTP request: %w", err) - } - - req.Header.Add("Content-Type", "application/json") - username, password := r.credentials.GetCredentials() - if username == nil || password == nil { - return errors.New("username or password not set for NGINX Plus usage reporting; unable to send reports." + - " Ensure that the usage Secret exists and the username and password are set") - } - req.SetBasicAuth(string(username), string(password)) - - client := http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: r.insecureSkipVerify, //nolint:gosec // used for testing - }, - }, - } - - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("error sending usage report request: %w", err) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("failed to read the response body: %w", err) - } - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("non-200 response: %v; response body: %v", resp.StatusCode, string(body)) - } - - return nil -} diff --git a/internal/mode/static/usage/reporter_test.go b/internal/mode/static/usage/reporter_test.go deleted file mode 100644 index 60db79f313..0000000000 --- a/internal/mode/static/usage/reporter_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package usage - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "testing" - - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" -) - -func TestReport(t *testing.T) { - t.Parallel() - g := NewWithT(t) - - data := ClusterDetails{ - Metadata: Metadata{ - UID: "12345abcde", - DisplayName: "my-cluster", - }, - NodeCount: 9, - PodDetails: PodDetails{ - CurrentPodCounts: CurrentPodsCount{ - PodCount: 12, - }, - }, - } - - secret := &v1.Secret{ - Data: map[string][]byte{ - "username": []byte("user"), - "password": []byte("pass"), - }, - } - - store := NewUsageSecret() - store.Set(secret) - - server := httptest.NewServer( - http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var reqData ClusterDetails - g.Expect(json.NewDecoder(r.Body).Decode(&reqData)).To(Succeed()) - g.Expect(reqData).To(Equal(data)) - - g.Expect(r.URL.Path).To(Equal(fmt.Sprintf("%s/%s", apiBasePath, data.Metadata.UID))) - g.Expect(r.Method).To(Equal(http.MethodPut)) - user, pass, ok := r.BasicAuth() - g.Expect(ok).To(BeTrue()) - g.Expect(user).To(Equal("user")) - g.Expect(pass).To(Equal("pass")) - - contentType, ok := r.Header["Content-Type"] - g.Expect(ok).To(BeTrue()) - g.Expect(contentType[0]).To(Equal("application/json")) - - w.WriteHeader(http.StatusOK) - }), - ) - defer server.Close() - - insecureSkipVerify := false - reporter, err := NewNIMReporter(store, server.URL, insecureSkipVerify) - g.Expect(err).ToNot(HaveOccurred()) - - g.Expect(reporter.Report(context.Background(), data)).To(Succeed()) -} - -func TestReport_NoCredentials(t *testing.T) { - t.Parallel() - g := NewWithT(t) - insecureSkipVerify := false - reporter, err := NewNIMReporter(NewUsageSecret(), "", insecureSkipVerify) - g.Expect(err).ToNot(HaveOccurred()) - - err = reporter.Report(context.Background(), ClusterDetails{}) - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("username or password not set")) -} - -func TestReport_ServerError(t *testing.T) { - t.Parallel() - g := NewWithT(t) - - server := httptest.NewServer( - http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - }), - ) - defer server.Close() - - secret := &v1.Secret{ - Data: map[string][]byte{ - "username": []byte("user"), - "password": []byte("pass"), - }, - } - - store := NewUsageSecret() - store.Set(secret) - - insecureSkipVerify := false - reporter, err := NewNIMReporter(store, server.URL, insecureSkipVerify) - g.Expect(err).ToNot(HaveOccurred()) - - err = reporter.Report(context.Background(), ClusterDetails{}) - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("non-200 response")) -} diff --git a/internal/mode/static/usage/secret.go b/internal/mode/static/usage/secret.go deleted file mode 100644 index 01d333cbdd..0000000000 --- a/internal/mode/static/usage/secret.go +++ /dev/null @@ -1,48 +0,0 @@ -package usage - -import ( - "sync" - - v1 "k8s.io/api/core/v1" -) - -// Secret implements the SecretStorer interface. -type Secret struct { - secret *v1.Secret - lock *sync.Mutex -} - -// NewUsageSecret creates a new Secret wrapper. -func NewUsageSecret() *Secret { - return &Secret{ - lock: &sync.Mutex{}, - } -} - -// Set stores the updated Secre. -func (s *Secret) Set(secret *v1.Secret) { - s.lock.Lock() - defer s.lock.Unlock() - - s.secret = secret -} - -// Delete nullifies the Secret value. -func (s *Secret) Delete() { - s.lock.Lock() - defer s.lock.Unlock() - - s.secret = nil -} - -// GetCredentials returns the base64 encoded username and password from the Secret. -func (s *Secret) GetCredentials() ([]byte, []byte) { - s.lock.Lock() - defer s.lock.Unlock() - - if s.secret != nil { - return s.secret.Data["username"], s.secret.Data["password"] - } - - return nil, nil -} diff --git a/internal/mode/static/usage/secret_test.go b/internal/mode/static/usage/secret_test.go deleted file mode 100644 index 82a5abd886..0000000000 --- a/internal/mode/static/usage/secret_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package usage - -import ( - "testing" - - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestSet(t *testing.T) { - t.Parallel() - store := NewUsageSecret() - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "secret", - Namespace: "custom", - }, - } - - g := NewWithT(t) - g.Expect(store.secret).To(BeNil()) - - store.Set(secret) - g.Expect(store.secret).To(Equal(secret)) -} - -func TestDelete(t *testing.T) { - t.Parallel() - store := NewUsageSecret() - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "secret", - Namespace: "custom", - }, - } - - g := NewWithT(t) - store.Set(secret) - g.Expect(store.secret).To(Equal(secret)) - - store.Delete() - g.Expect(store.secret).To(BeNil()) -} - -func TestGetCredentials(t *testing.T) { - t.Parallel() - store := NewUsageSecret() - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "secret", - Namespace: "custom", - }, - Data: map[string][]byte{ - "username": []byte("user"), - "password": []byte("pass"), - }, - } - - g := NewWithT(t) - - user, pass := store.GetCredentials() - g.Expect(user).To(BeNil()) - g.Expect(pass).To(BeNil()) - - store.Set(secret) - - user, pass = store.GetCredentials() - g.Expect(user).To(Equal([]byte("user"))) - g.Expect(pass).To(Equal([]byte("pass"))) -} diff --git a/internal/mode/static/usage/usagefakes/fake_credentials_getter.go b/internal/mode/static/usage/usagefakes/fake_credentials_getter.go deleted file mode 100644 index 317565732a..0000000000 --- a/internal/mode/static/usage/usagefakes/fake_credentials_getter.go +++ /dev/null @@ -1,103 +0,0 @@ -// Code generated by counterfeiter. DO NOT EDIT. -package usagefakes - -import ( - "sync" -) - -type FakeCredentialsGetter struct { - GetCredentialsStub func() ([]byte, []byte) - getCredentialsMutex sync.RWMutex - getCredentialsArgsForCall []struct { - } - getCredentialsReturns struct { - result1 []byte - result2 []byte - } - getCredentialsReturnsOnCall map[int]struct { - result1 []byte - result2 []byte - } - invocations map[string][][]interface{} - invocationsMutex sync.RWMutex -} - -func (fake *FakeCredentialsGetter) GetCredentials() ([]byte, []byte) { - fake.getCredentialsMutex.Lock() - ret, specificReturn := fake.getCredentialsReturnsOnCall[len(fake.getCredentialsArgsForCall)] - fake.getCredentialsArgsForCall = append(fake.getCredentialsArgsForCall, struct { - }{}) - stub := fake.GetCredentialsStub - fakeReturns := fake.getCredentialsReturns - fake.recordInvocation("GetCredentials", []interface{}{}) - fake.getCredentialsMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeCredentialsGetter) GetCredentialsCallCount() int { - fake.getCredentialsMutex.RLock() - defer fake.getCredentialsMutex.RUnlock() - return len(fake.getCredentialsArgsForCall) -} - -func (fake *FakeCredentialsGetter) GetCredentialsCalls(stub func() ([]byte, []byte)) { - fake.getCredentialsMutex.Lock() - defer fake.getCredentialsMutex.Unlock() - fake.GetCredentialsStub = stub -} - -func (fake *FakeCredentialsGetter) GetCredentialsReturns(result1 []byte, result2 []byte) { - fake.getCredentialsMutex.Lock() - defer fake.getCredentialsMutex.Unlock() - fake.GetCredentialsStub = nil - fake.getCredentialsReturns = struct { - result1 []byte - result2 []byte - }{result1, result2} -} - -func (fake *FakeCredentialsGetter) GetCredentialsReturnsOnCall(i int, result1 []byte, result2 []byte) { - fake.getCredentialsMutex.Lock() - defer fake.getCredentialsMutex.Unlock() - fake.GetCredentialsStub = nil - if fake.getCredentialsReturnsOnCall == nil { - fake.getCredentialsReturnsOnCall = make(map[int]struct { - result1 []byte - result2 []byte - }) - } - fake.getCredentialsReturnsOnCall[i] = struct { - result1 []byte - result2 []byte - }{result1, result2} -} - -func (fake *FakeCredentialsGetter) Invocations() map[string][][]interface{} { - fake.invocationsMutex.RLock() - defer fake.invocationsMutex.RUnlock() - fake.getCredentialsMutex.RLock() - defer fake.getCredentialsMutex.RUnlock() - copiedInvocations := map[string][][]interface{}{} - for key, value := range fake.invocations { - copiedInvocations[key] = value - } - return copiedInvocations -} - -func (fake *FakeCredentialsGetter) recordInvocation(key string, args []interface{}) { - fake.invocationsMutex.Lock() - defer fake.invocationsMutex.Unlock() - if fake.invocations == nil { - fake.invocations = map[string][][]interface{}{} - } - if fake.invocations[key] == nil { - fake.invocations[key] = [][]interface{}{} - } - fake.invocations[key] = append(fake.invocations[key], args) -} diff --git a/internal/mode/static/usage/usagefakes/fake_reporter.go b/internal/mode/static/usage/usagefakes/fake_reporter.go deleted file mode 100644 index b33a8759aa..0000000000 --- a/internal/mode/static/usage/usagefakes/fake_reporter.go +++ /dev/null @@ -1,114 +0,0 @@ -// Code generated by counterfeiter. DO NOT EDIT. -package usagefakes - -import ( - "context" - "sync" - - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/usage" -) - -type FakeReporter struct { - ReportStub func(context.Context, usage.ClusterDetails) error - reportMutex sync.RWMutex - reportArgsForCall []struct { - arg1 context.Context - arg2 usage.ClusterDetails - } - reportReturns struct { - result1 error - } - reportReturnsOnCall map[int]struct { - result1 error - } - invocations map[string][][]interface{} - invocationsMutex sync.RWMutex -} - -func (fake *FakeReporter) Report(arg1 context.Context, arg2 usage.ClusterDetails) error { - fake.reportMutex.Lock() - ret, specificReturn := fake.reportReturnsOnCall[len(fake.reportArgsForCall)] - fake.reportArgsForCall = append(fake.reportArgsForCall, struct { - arg1 context.Context - arg2 usage.ClusterDetails - }{arg1, arg2}) - stub := fake.ReportStub - fakeReturns := fake.reportReturns - fake.recordInvocation("Report", []interface{}{arg1, arg2}) - fake.reportMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeReporter) ReportCallCount() int { - fake.reportMutex.RLock() - defer fake.reportMutex.RUnlock() - return len(fake.reportArgsForCall) -} - -func (fake *FakeReporter) ReportCalls(stub func(context.Context, usage.ClusterDetails) error) { - fake.reportMutex.Lock() - defer fake.reportMutex.Unlock() - fake.ReportStub = stub -} - -func (fake *FakeReporter) ReportArgsForCall(i int) (context.Context, usage.ClusterDetails) { - fake.reportMutex.RLock() - defer fake.reportMutex.RUnlock() - argsForCall := fake.reportArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeReporter) ReportReturns(result1 error) { - fake.reportMutex.Lock() - defer fake.reportMutex.Unlock() - fake.ReportStub = nil - fake.reportReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeReporter) ReportReturnsOnCall(i int, result1 error) { - fake.reportMutex.Lock() - defer fake.reportMutex.Unlock() - fake.ReportStub = nil - if fake.reportReturnsOnCall == nil { - fake.reportReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.reportReturnsOnCall[i] = struct { - result1 error - }{result1} -} - -func (fake *FakeReporter) Invocations() map[string][][]interface{} { - fake.invocationsMutex.RLock() - defer fake.invocationsMutex.RUnlock() - fake.reportMutex.RLock() - defer fake.reportMutex.RUnlock() - copiedInvocations := map[string][][]interface{}{} - for key, value := range fake.invocations { - copiedInvocations[key] = value - } - return copiedInvocations -} - -func (fake *FakeReporter) recordInvocation(key string, args []interface{}) { - fake.invocationsMutex.Lock() - defer fake.invocationsMutex.Unlock() - if fake.invocations == nil { - fake.invocations = map[string][][]interface{}{} - } - if fake.invocations[key] == nil { - fake.invocations[key] = [][]interface{}{} - } - fake.invocations[key] = append(fake.invocations[key], args) -} - -var _ usage.Reporter = new(FakeReporter) diff --git a/site/content/how-to/monitoring/troubleshooting.md b/site/content/how-to/monitoring/troubleshooting.md index 7ea6eb0285..b5159d73a7 100644 --- a/site/content/how-to/monitoring/troubleshooting.md +++ b/site/content/how-to/monitoring/troubleshooting.md @@ -103,14 +103,6 @@ You can see logs for a crashed or killed container by adding the `-p` flag to th kubectl -n nginx-gateway logs -c nginx-gateway | grep error ``` - For example, an error message when telemetry is not enabled for NGINX Plus installations: - - ```text - kubectl logs -n nginx-gateway nginx-gateway-nginx-gateway-fabric-77f8746996-j6z6v | grep error - Defaulted container "nginx-gateway" out of: nginx-gateway, nginx - {"level":"error","ts":"2024-06-13T18:22:16Z","logger":"usageReporter","msg":"Usage reporting must be enabled when using NGINX Plus; redeploy with usage reporting enabled","error":"usage reporting not enabled","stacktrace":"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static.createUsageWarningJob.func1\n\tgithub.com/nginxinc/nginx-gateway-fabric/internal/mode/static/manager.go:616\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:259\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:226\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:227\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:204\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:259\ngithub.com/nginxinc/nginx-gateway-fabric/internal/framework/runnables.(*CronJob).Start\n\tgithub.com/nginxinc/nginx-gateway-fabric/internal/framework/runnables/cronjob.go:53\nsigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func1\n\tsigs.k8s.io/controller-runtime@v0.18.4/pkg/manager/runnable_group.go:226"} - ``` - For the _nginx_ container you can `grep` for various [error](https://nginx.org/en/docs/ngx_core_module.html#error_log) logs. For example, to search for all logs logged at the `emerg` level: ```shell @@ -300,7 +292,7 @@ Verify that the port number (for example, `8080`) matches the port number you ha | Startup | NGINX Gateway Fabric fails to start. | Check logs for _nginx_ and _nginx-gateway_ containers. | Readiness probe failed. | | Resources not configured | Status missing on resources. | Check referenced resources. | Referenced resources do not belong to NGINX Gateway Fabric. | | NGINX errors | Reload failures on NGINX | Fix permissions for control plane. | Security context not configured. | -| Usage reporting | Errors logs related to usage reporting | Enable usage reporting. Refer to [Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) | Usage reporting disabled. | +| NGINX Plus errors | Failure to start; traffic interruptions | Set up the [NGINX Plus JWT]({{< relref "installation/nginx-plus-jwt.md" >}}) | License is not configured or has expired. | | Client Settings | Request entity too large error | Adjust client settings. Refer to [Client Settings Policy]({{< relref "../traffic-management/client-settings.md" >}}) | Payload is greater than the [`client_max_body_size`](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) value.| {{< /bootstrap-table >}} @@ -373,13 +365,23 @@ To **resolve** this issue you will need to set `allowPrivilegeEscalation` to `tr - If using Helm, you can set the `nginxGateway.securityContext.allowPrivilegeEscalation` value. - If using the manifests directly, you can update this field under the `nginx-gateway` container's `securityContext`. -##### Usage Reporting errors +##### NGINX Plus failure to start or traffic interruptions + +Beginning with NGINX Gateway Fabric 1.5.0, NGINX Plus requires a valid JSON Web Token (JWT) to run. If this is not set up properly, or your JWT token has expired, you may see errors in the NGINX logs that look like the following: + +```text +nginx: [error] invalid license token +``` -If using NGINX Gateway Fabric with NGINX Plus as the data plane, you will see the following error in the _nginx-gateway_ logs if you have not enabled Usage Reporting: +```text +nginx: [emerg] License file is required. Download JWT license from MyF5 and configure its location... +``` -`usage reporting not enabled` +```text +nginx: [emerg] license expired +``` -To **resolve** this issue, enable Usage Reporting by following the [Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) guide. +These errors could prevent NGINX Plus from starting or prevent traffic from flowing. To fix these issues, see the [NGINX Plus JWT]({{< relref "installation/nginx-plus-jwt.md" >}}) guide. ##### 413 Request Entity Too Large diff --git a/site/content/includes/installation/nginx-plus/docker-registry-secret.md b/site/content/includes/installation/nginx-plus/docker-registry-secret.md new file mode 100644 index 0000000000..b909f8f22e --- /dev/null +++ b/site/content/includes/installation/nginx-plus/docker-registry-secret.md @@ -0,0 +1,19 @@ +--- +docs: "DOCS-000" +--- + +{{< note >}} If you would rather pull the NGINX Plus image and push to a private registry, you can skip this specific step and instead follow [this step]({{}}). {{< /note >}} + +If the `nginx-gateway` namespace does not yet exist, create it: + +```shell +kubectl create namespace nginx-gateway +``` + +Create a Kubernetes `docker-registry` secret type using the contents of the JWT as the username and `none` for password (as the password is not used). The name of the docker server is `private-registry.nginx.com`. + +```shell +kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username= --docker-password=none -n nginx-gateway +``` + +It is important that the `--docker-username=` contains the contents of the token and is not pointing to the token itself. When you copy the contents of the JWT, ensure there are no additional characters such as extra whitespaces. This can invalidate the token, causing 401 errors when trying to authenticate to the registry. diff --git a/site/content/includes/installation/nginx-plus/download-jwt.md b/site/content/includes/installation/nginx-plus/download-jwt.md new file mode 100644 index 0000000000..d89c65a43b --- /dev/null +++ b/site/content/includes/installation/nginx-plus/download-jwt.md @@ -0,0 +1,10 @@ +--- +docs: "DOCS-000" +--- + +1. Log in to [MyF5](https://my.f5.com/manage/s/). +2. Go to **My Products & Plans > Subscriptions** to see your active subscriptions. +3. Find your NGINX products or services subscription, and select the **Subscription ID** for details. +4. Download the **JSON Web Token (JWT)** from the subscription page. + +{{< note >}} The Connectivity Stack for Kubernetes JWT does not work with NGINX Plus reporting. A regular NGINX Plus instance JWT must be used. {{< /note >}} diff --git a/site/content/includes/installation/nginx-plus/nginx-plus-secret.md b/site/content/includes/installation/nginx-plus/nginx-plus-secret.md new file mode 100644 index 0000000000..1a5beb4746 --- /dev/null +++ b/site/content/includes/installation/nginx-plus/nginx-plus-secret.md @@ -0,0 +1,13 @@ +--- +docs: "DOCS-000" +--- + +Place the JWT in a file called `license.jwt`. Create a Kubernetes Secret using the contents of the JWT file. + +```shell +kubectl create secret generic nplus-license --from-file license.jwt -n nginx-gateway +``` + +You can now delete the `license.jwt` file. + +If you need to update the JWT at any time, update the `license.jwt` field in the Secret using `kubectl edit` and apply the changes. diff --git a/site/content/installation/ngf-images/building-the-images.md b/site/content/installation/building-the-images.md similarity index 99% rename from site/content/installation/ngf-images/building-the-images.md rename to site/content/installation/building-the-images.md index 9276114955..ed03e099f1 100644 --- a/site/content/installation/ngf-images/building-the-images.md +++ b/site/content/installation/building-the-images.md @@ -1,6 +1,6 @@ --- title: "Build NGINX Gateway Fabric and NGINX images" -weight: 300 +weight: 500 toc: true docs: "DOCS-1431" --- diff --git a/site/content/installation/installing-ngf/helm.md b/site/content/installation/installing-ngf/helm.md index cc3bded585..763a1a9e39 100644 --- a/site/content/installation/installing-ngf/helm.md +++ b/site/content/installation/installing-ngf/helm.md @@ -9,16 +9,37 @@ docs: "DOCS-1430" Learn how to install, upgrade, and uninstall NGINX Gateway Fabric in a Kubernetes cluster using Helm. +{{< important >}} NGINX Plus users that are upgrading from version 1.4.0 to 1.5.0 need to install an NGINX Plus JWT +Secret before upgrading. Follow the steps in the [Before you begin](#before-you-begin) section to create the Secret. If you use a different name than the default `nplus-license` name, specify the Secret name by setting `--set nginx.usage.secretName=` when running `helm upgrade`. {{< /important >}} + ## Before you begin To complete this guide, you'll need to install: - [kubectl](https://kubernetes.io/docs/tasks/tools/), a command-line tool for managing Kubernetes clusters. - [Helm 3.0 or later](https://helm.sh/docs/intro/install/), for deploying and managing applications on Kubernetes. -- If you’d like to use NGINX Plus: - 1. To pull from the F5 Container registry, configure a docker registry secret using your JWT token from the MyF5 portal by following the instructions from [here]({{}}). Make sure to specify the secret using `nginxGateway.serviceAccount.imagePullSecret` or `nginxGateway.serviceAccount.imagePullSecrets` parameter. - 1. Alternatively, pull an NGINX Gateway Fabric image with NGINX Plus and push it to your private registry by following the instructions from [here]({{}}). - 1. Update the `nginxGateway.image.repository` field of the `values.yaml` accordingly. + +{{< important >}} If you’d like to use NGINX Plus, some additional setup is also required: {{}} +
+NGINX Plus JWT setup + +{{}} + +### 1. Download the JWT from MyF5 + +{{}} + +### 2. Create the Docker Registry Secret + +{{}} + +### 3. Create the NGINX Plus Secret + +{{}} + +{{< note >}} For more information on why this is needed and additional configuration options, including how to report to NGINX Instance Manager instead, see the [NGINX Plus Image and JWT Requirement]({{< relref "installation/nginx-plus-jwt.md" >}}) document. {{< /note >}} + +
## Deploy NGINX Gateway Fabric @@ -28,7 +49,7 @@ To complete this guide, you'll need to install: ### Install from the OCI registry -{{}} +{{}} {{%tab name="NGINX"%}} @@ -42,48 +63,58 @@ helm install ngf oci://ghcr.io/nginxinc/charts/nginx-gateway-fabric --create-nam {{%tab name="NGINX Plus"%}} -{{< note >}}If applicable, replace the F5 Container registry `private-registry.nginx.com` with your internal registry for your NGINX Plus image, and replace `nginx-plus-registry-secret` with your Secret name containing the registry credentials.{{< /note >}} +{{< note >}} If applicable, replace the F5 Container registry `private-registry.nginx.com` with your internal registry for your NGINX Plus image, and replace `nginx-plus-registry-secret` with your Secret name containing the registry credentials. If your NGINX Plus JWT Secret has a different name than the default `nplus-license`, then define that name using the `nginx.usage.secretName` flag. {{< /note >}} -{{< important >}}Ensure that you [Enable Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) when installing.{{< /important >}} +To install the latest stable release of NGINX Gateway Fabric in the **nginx-gateway** namespace, run the following command: ```shell helm install ngf oci://ghcr.io/nginxinc/charts/nginx-gateway-fabric --set nginx.image.repository=private-registry.nginx.com/nginx-gateway-fabric/nginx-plus --set nginx.plus=true --set serviceAccount.imagePullSecret=nginx-plus-registry-secret --create-namespace -n nginx-gateway ``` +{{% /tab %}} + +{{}} + `ngf` is the name of the release, and can be changed to any name you want. This name is added as a prefix to the Deployment name. If the namespace already exists, you can omit the optional `--create-namespace` flag. If you want the latest version from the **main** branch, add `--version 0.0.0-edge` to your install command. -You can also use the certificate and key from the MyF5 portal and the Docker registry API to list the available image tags for NGINX Plus, for example: +To wait for the Deployment to be ready, you can either add the `--wait` flag to the `helm install` command, or run the following after installing: ```shell - $ curl https://private-registry.nginx.com/v2/nginx-gateway-fabric/nginx-plus/tags/list --key --cert | jq - - { - "name": "nginx-gateway-fabric/nginx-plus", - "tags": ["edge"] - } +kubectl wait --timeout=5m -n nginx-gateway deployment/ngf-nginx-gateway-fabric --for=condition=Available ``` -{{% /tab %}} +### Install from sources {#install-from-sources} -{{}} -To wait for the Deployment to be ready, you can either add the `--wait` flag to the `helm install` command, or run the following after installing: +{{}} + +{{}} + +{{%tab name="NGINX"%}} + +To install the chart into the **nginx-gateway** namespace, run the following command: ```shell -kubectl wait --timeout=5m -n nginx-gateway deployment/ngf-nginx-gateway-fabric --for=condition=Available +helm install ngf . --create-namespace -n nginx-gateway ``` -### Install from sources {#install-from-sources} +{{% /tab %}} -1. {{}} +{{%tab name="NGINX Plus"%}} -2. To install the chart into the **nginx-gateway** namespace, run the following command. +{{< note >}} If applicable, replace the F5 Container registry `private-registry.nginx.com` with your internal registry for your NGINX Plus image, and replace `nginx-plus-registry-secret` with your Secret name containing the registry credentials. If your NGINX Plus JWT Secret has a different name than the default `nplus-license`, then define that name using the `nginx.usage.secretName` flag. {{< /note >}} - ```shell - helm install ngf . --create-namespace -n nginx-gateway - ``` +To install the chart into the **nginx-gateway** namespace, run the following command: + +```shell +helm install ngf . --set nginx.image.repository=private-registry.nginx.com/nginx-gateway-fabric/nginx-plus --set nginx.plus=true --set serviceAccount.imagePullSecret=nginx-plus-registry-secret --create-namespace -n nginx-gateway +``` + +{{% /tab %}} + +{{}} `ngf` is the name of the release, and can be changed to any name you want. This name is added as a prefix to the Deployment name. @@ -178,6 +209,9 @@ To upgrade the CRDs, take the following steps: ### Upgrade NGINX Gateway Fabric release +{{}}NGINX Plus users that are upgrading from version 1.4.0 to 1.5.0 need to install an NGINX Plus JWT +Secret before upgrading. Follow the steps in the [Before you begin](#before-you-begin) section to create the Secret. If you use a different name than the default `nplus-license` name, specify the Secret name by setting `--set nginx.usage.secretName=` when running `helm upgrade`.{{}} + #### Upgrade from the OCI registry - To upgrade to the latest stable release of NGINX Gateway Fabric, run: @@ -206,7 +240,7 @@ To upgrade the CRDs, take the following steps: {{< note >}}If applicable, replace the F5 Container registry `private-registry.nginx.com` with your internal registry for your NGINX Plus image, and replace `nginx-plus-registry-secret` with your Secret name containing the registry credentials.{{< /note >}} - {{< important >}}Ensure that you [Enable Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) when installing.{{< /important >}} + {{< important >}}Ensure that you [Create the required JWT Secrets]({{< relref "installation/nginx-plus-jwt.md" >}}) before installing.{{< /important >}} ```shell helm upgrade ngf oci://ghcr.io/nginxinc/charts/nginx-gateway-fabric --set nginx.image.repository=private-registry.nginx.com/nginx-gateway-fabric/nginx-plus --set nginx.plus=true --set serviceAccount.imagePullSecret=nginx-plus-registry-secret -n nginx-gateway diff --git a/site/content/installation/installing-ngf/manifests.md b/site/content/installation/installing-ngf/manifests.md index 8ed98732ed..c2eff51048 100644 --- a/site/content/installation/installing-ngf/manifests.md +++ b/site/content/installation/installing-ngf/manifests.md @@ -9,21 +9,41 @@ docs: "DOCS-1429" Learn how to install, upgrade, and uninstall NGINX Gateway Fabric using Kubernetes manifests. +{{< important >}} NGINX Plus users that are upgrading from version 1.4.0 to 1.5.0 need to install an NGINX Plus JWT +Secret before upgrading. Follow the steps in the [Before you begin](#before-you-begin) section to create the Secret, which is referenced in the updated deployment manifest for the newest version. {{< /important >}} + ## Before you begin To complete this guide, you'll need to install: - [kubectl](https://kubernetes.io/docs/tasks/tools/), a command-line interface for managing Kubernetes clusters. +{{< important >}} If you’d like to use NGINX Plus, some additional setup is also required: {{}} +
+NGINX Plus JWT setup + +{{}} + +### 1. Download the JWT from MyF5 + +{{}} + +### 2. Create the Docker Registry Secret + +{{}} + +### 3. Create the NGINX Plus Secret + +{{}} + +{{< note >}} For more information on why this is needed and additional configuration options, including how to report to NGINX Instance Manager instead, see the [NGINX Plus Image and JWT Requirement]({{< relref "installation/nginx-plus-jwt.md" >}}) document. {{< /note >}} + +
+ ## Deploy NGINX Gateway Fabric Deploying NGINX Gateway Fabric with Kubernetes manifests takes only a few steps. With manifests, you can configure your deployment exactly how you want. Manifests also make it easy to replicate deployments across environments or clusters, ensuring consistency. -- If you’d like to use NGINX Plus: - 1. To pull from the F5 Container registry, configure a docker registry secret using your JWT token from the MyF5 portal by following the instructions from [here](https://docs.nginx.com/nginx-gateway-fabric/installation/ngf-images/jwt-token-docker-secret). Make sure to specify the secret in the `imagePullSecrets` field of the `nginx-gateway` ServiceAccount. - 1. Alternatively, pull an NGINX Gateway Fabric image with NGINX Plus and push it to your private registry by following the instructions from [here]({{}}). - 1. Update the nginx container's `image` field of the `nginx-gateway` Deployment accordingly. - ### 1. Install the Gateway API resources {{}} @@ -44,7 +64,7 @@ kubectl apply -f https://raw.githubusercontent.com/nginxinc/nginx-gateway-fabric ### 3. Deploy NGINX Gateway Fabric -{{}}By default, NGINX Gateway Fabric is installed in the **nginx-gateway** namespace. You can deploy in another namespace by modifying the manifest files.{{}} +{{< note >}} By default, NGINX Gateway Fabric is installed in the **nginx-gateway** namespace. You can deploy in another namespace by modifying the manifest files. {{< /note >}} {{}} @@ -81,10 +101,8 @@ kubectl apply -f https://raw.githubusercontent.com/nginxinc/nginx-gateway-fabric {{%tab name="NGINX Plus"%}} Deploys NGINX Gateway Fabric with NGINX Plus. The image is pulled from the -NGINX Plus Docker registry, and the `imagePullSecretName` is the name of the secret to use to pull the image. -The secret must be created in the same namespace as the NGINX Gateway Fabric deployment. - -{{< important >}}Ensure that you [Enable Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) and update the necessary fields before applying.{{< /important >}} +NGINX Plus Docker registry, and the `imagePullSecretName` is the name of the Secret to use to pull the image. +The NGINX Plus JWT Secret used to run NGINX Plus is also specified in a volume mount and the `--usage-report-secret` parameter. These Secrets are created as part of the [Before you begin](#before-you-begin) section. ```shell kubectl apply -f https://raw.githubusercontent.com/nginxinc/nginx-gateway-fabric/v1.4.0/deploy/nginx-plus/deploy.yaml @@ -100,21 +118,21 @@ Deploys NGINX Gateway Fabric with NGINX OSS and experimental features. kubectl apply -f https://raw.githubusercontent.com/nginxinc/nginx-gateway-fabric/v1.4.0/deploy/experimental/deploy.yaml ``` -{{}}Requires the Gateway APIs installed from the experimental channel.{{}} +{{< note >}} Requires the Gateway APIs installed from the experimental channel. {{< /note >}} {{% /tab %}} {{%tab name="NGINX Plus Experimental"%}} -Deploys NGINX Gateway Fabric with NGINX Plus and experimental features. The image is pulled from the NGINX Plus Docker registry, and the `imagePullSecretName` is the name of the secret to use to pull the image. The secret must be created in the same namespace as the NGINX Gateway Fabric deployment. - -{{< important >}}Ensure that you [Enable Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) and update the necessary fields before applying.{{< /important >}} +Deploys NGINX Gateway Fabric with NGINX Plus and experimental features. The image is pulled from the +NGINX Plus Docker registry, and the `imagePullSecretName` is the name of the Secret to use to pull the image. +The NGINX Plus JWT Secret used to run NGINX Plus is also specified in a volume mount and the `--usage-report-secret` parameter. These Secrets are created as part of the [Before you begin](#before-you-begin) section. ```shell kubectl apply -f https://raw.githubusercontent.com/nginxinc/nginx-gateway-fabric/v1.4.0/deploy/nginx-plus-experimental/deploy.yaml ``` -{{}}Requires the Gateway APIs installed from the experimental channel.{{}} +{{< note >}} Requires the Gateway APIs installed from the experimental channel. {{< /note >}} {{% /tab %}} @@ -161,6 +179,9 @@ nginx-gateway-5d4f4c7db7-xk2kq 2/2 Running 0 112s ## Upgrade NGINX Gateway Fabric +{{< important >}} NGINX Plus users that are upgrading from version 1.4.0 to 1.5.0 need to install an NGINX Plus JWT +Secret before upgrading. Follow the steps in the [Before you begin](#before-you-begin) section to create the Secret, which is referenced in the updated deployment manifest for the newest version. {{< /important >}} + {{}}For guidance on zero downtime upgrades, see the [Delay Pod Termination](#configure-delayed-pod-termination-for-zero-downtime-upgrades) section below.{{}} To upgrade NGINX Gateway Fabric and get the latest features and improvements, take the following steps: diff --git a/site/content/installation/ngf-images/_index.md b/site/content/installation/ngf-images/_index.md deleted file mode 100644 index f4eb6b6e79..0000000000 --- a/site/content/installation/ngf-images/_index.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "NGINX Gateway Fabric Images" -description: -weight: 300 -linkTitle: "Images" -menu: - docs: - parent: Installation ---- diff --git a/site/content/installation/ngf-images/jwt-token-docker-secret.md b/site/content/installation/ngf-images/jwt-token-docker-secret.md deleted file mode 100644 index 3fddec3cc2..0000000000 --- a/site/content/installation/ngf-images/jwt-token-docker-secret.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -title: "Get the NGINX Plus image using JWT" -weight: 100 -doctypes: ["install"] -toc: true -docs: "DOCS-1432" ---- - -## Overview - -This document describes how to use a JWT token to get an NGINX Plus image for NGINX Gateway Fabric from the F5 Docker registry. - -Follow the steps in this document to pull the NGINX Plus image for NGINX Gateway Fabric from the F5 Docker registry into your Kubernetes cluster using your JWT token. To list available image tags using the Docker registry API, you will also need to download its certificate and key from [MyF5](https://my.f5.com). - -{{}}An NGINX Plus subscription will not work with these instructions. For NGINX Gateway Fabric, you must have an Connectivity Stack for Kubernetes subscription.{{}} - -## Before you begin - -You will need the following items from [MyF5](https://my.f5.com) for these instructions: - -1. A JWT Access Token for NGINX Gateway Fabric from an active Connectivity Stack for Kubernetes subscription (Per instance). -1. The certificate (**nginx-repo.crt**) and key (**nginx-repo.key**) for each NGINX Gateway Fabric instance. - -## Get the Credentials - -1. Log into the [MyF5 Portal](https://my.f5.com/), navigate to your subscription details, and download the required certificate, key and JWT files. - -## Using the JWT token in a Docker Config Secret - -1. Create a Kubernetes `docker-registry` secret type on the cluster, using the contents of the JWT token as the username and `none` for password (as the password is not used). The name of the docker server is `private-registry.nginx.com`. - - ```shell - kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username= --docker-password=none [-n nginx-gateway] - ``` - - It is important that the `--docker-username=` contains the contents of the token and is not pointing to the token itself. When you copy the contents of the JWT token, ensure there are no additional characters such as extra whitespaces. This can invalidate the token, causing 401 errors when trying to authenticate to the registry. - -1. Inspect and verify the details of the created secret by running: - - ```shell - kubectl get secret nginx-plus-registry-secret --output=yaml - ``` - -{{< include "installation/jwt-password-note.md" >}} - -## Install NGINX Gateway Fabric - -Please refer to [Installing NGINX Gateway Fabric]({{< relref "installation/installing-ngf" >}}) - - -## Pulling an image for local use - -To pull an image for local use, use this command: - -```shell -docker login private-registry.nginx.com --username= --password=none -``` - -Replace the contents of `` with the contents of the JWT token itself. -Once you have successfully pulled the image, you can tag it as needed, then push it to a different container registry. - - -## Alternative installation options - -There are alternative ways to get an NGINX Plus image for NGINX Gateway Fabric: - -- [Build the Gateway Fabric image]({{}}) describes how to use the source code with an NGINX Plus subscription certificate and key to build an image. diff --git a/site/content/installation/ngf-images/pulling-ngf-image.md b/site/content/installation/ngf-images/pulling-ngf-image.md deleted file mode 100644 index 030770c5e9..0000000000 --- a/site/content/installation/ngf-images/pulling-ngf-image.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: "Push an NGINX Plus image to a private registry" -weight: 200 -doctypes: ["install"] -toc: true -docs: "DOCS-1433" ---- - -## Overview - -This document describes how to pull a NGINX Plus image for NGINX Gateway Fabric from the official F5 Docker registry and upload it to your private registry - -## Before you begin - -Before you start, you'll need these installed on your machine: - -- [Docker v18.09 or higher](https://docs.docker.com/engine/release-notes/18.09/). -- The certificate (**nginx-repo.crt**) and key (**nginx-repo.key**) for a Connectivity Stack for Kubernetes subscription, obtainable from [MyF5l](https://my.f5.com) An NGINX Plus certificate and key will not work. - -## Configuring Docker for the F5 Container Registry - -To configure Docker to communicate with the F5 Container Registry, first create a folder containing your certificate and key files: - -```shell -mkdir -p /etc/docker/certs.d/private-registry.nginx.com -cp /etc/docker/certs.d/private-registry.nginx.com/client.cert -cp /etc/docker/certs.d/private-registry.nginx.com/client.key -``` - -If you are not using a Linux operating system, read the [Docker for Windows](https://docs.docker.com/desktop/faqs/windowsfaqs/#how-do-i-add-custom-ca-certificates) or [Docker for Mac](https://docs.docker.com/desktop/faqs/macfaqs/#add-custom-ca-certificates-server-side) instructions. For more details on Docker Engine security, you can refer to the [Docker Engine Security documentation](https://docs.docker.com/engine/security/). - - -## Pulling the image - -Once configured, you can now pull images from `private-registry.nginx.com`. To find your desired image, read the [Technical Specifications](https://github.com/nginxinc/nginx-gateway-fabric#technical-specifications). - -Run this command step to pull an image, replacing `` with the specific version you need, such as `1.4.0`. - - - ```shell - docker pull private-registry.nginx.com/nginx-gateway-fabric/nginx-plus:1.4.0 - ``` - -You can use the Docker registry API to list available image tags using your client certificate and key. The `jq` command is used to format the JSON output for easier reading. - -```shell -curl https://private-registry.nginx.com/nginx-gateway-fabric/nginx-plus/tags/list --key --cert | jq -``` - -```json -{ - "name": "nginx-gateway-fabric/nginx-plus", - "tags": [ - "edge", - "nightly" - ] -} -``` - - -Once you have pulled an image, you can tag it and push it to a private registry. - -1. Log into your private registry: - - ```shell - docker login - ``` - -1. Tag the image, replacing `` with your registry's path and `` with the version you're using: - - - ```shell - docker tag private-registry.nginx.com/nginx-gateway-fabric/nginx-plus: /nginx-gateway-fabric/nginx-plus: - docker push /nginx-gateway-fabric/nginx-plus: - ``` - - -## Troubleshooting - -If you encounter issues while following this guide, here are solutions to common problems: - -- **Certificate errors**: - - *Likely cause*: Incorrect certificate or key location, or using an NGINX Plus certificate. - - *Solution*: Check you have the correct NGINX Gateway Fabric certificate and key, their files are named correctly, and they are in the correct directory. - -- **Docker version compatibility** - - *Likely cause*: Outdated Docker version. - - *Solution*: Make sure you're running [Docker v18.09 or higher](https://docs.docker.com/engine/release-notes/18.09/), and upgrade if necessary. - -- **Can't pull the image** - - *Likely cause*: Mismatched image name or tag. - - *Solution*: Compare the image name and tag to the [Technical Specifications table](https://github.com/nginxinc/nginx-gateway-fabric?tab=readme-ov-file#technical-specifications). - -- **Failed to push to private registry** - - *Likely cause*: Not logged into your private registry or incorrect image tagging. - - *Solution*: Verify your login status and correct the image tag before pushing. Read the [Docker documentation](https://docs.docker.com/docker-hub/repos/) for more guidance. - - -## Alternative installation options - -There are alternative ways to get an NGINX Plus image for NGINX Gateway Fabric: - -- [Install by pulling a docker image]({{}}). -- [Build the Gateway Fabric image]({{}}) using the source code from the GitHub repository and your NGINX Plus subscription certificate and key. diff --git a/site/content/installation/nginx-plus-jwt.md b/site/content/installation/nginx-plus-jwt.md new file mode 100644 index 0000000000..1488b65ed1 --- /dev/null +++ b/site/content/installation/nginx-plus-jwt.md @@ -0,0 +1,240 @@ +--- +title: "NGINX Plus Image and JWT Requirement" +weight: 300 +toc: true +docs: "DOCS-000" +--- + +## Overview + +NGINX Gateway Fabric with NGINX Plus requires a valid JSON Web Token (JWT) to download the container image from the F5 registry. In addition, starting with version 1.5.0, this JWT token is also required to run NGINX Plus. + +This requirement is part of F5’s broader licensing program and aligns with industry best practices. The JWT will streamline subscription renewals and usage reporting, helping you manage your NGINX Plus subscription more efficiently. The [telemetry](#telemetry) data we collect helps us improve our products and services to better meet your needs. + +The JWT is required for validating your subscription and reporting telemetry data. For environments connected to the internet, telemetry is automatically sent to F5’s licensing endpoint. In offline environments, telemetry is routed through [NGINX Instance Manager](https://docs.nginx.com/nginx-management-suite/nim/). Usage is reported every hour and on startup whenever NGINX is reloaded. + +## Setting up the JWT + +The JWT needs to be configured before deploying NGINX Gateway Fabric. We'll store the JWT in two Kubernetes Secrets. One will be used for downloading the NGINX Plus container image, and the other for running NGINX Plus. + +{{< include "installation/jwt-password-note.md" >}} + +### Download the JWT from MyF5 + +{{< include "installation/nginx-plus/download-jwt.md" >}} + +### Docker Registry Secret + +{{< include "installation/nginx-plus/docker-registry-secret.md" >}} + +Provide the name of this Secret when installing NGINX Gateway Fabric: + +{{}} + +{{%tab name="Helm"%}} + +Specify the Secret name using the `serviceAccount.imagePullSecret` or `serviceAccount.imagePullSecrets` helm value. + +{{% /tab %}} + +{{%tab name="Manifests"%}} + +Specify the Secret name in the `imagePullSecrets` field of the `nginx-gateway` ServiceAccount. + +{{% /tab %}} + +{{}} + +### NGINX Plus Secret + +{{< include "installation/nginx-plus/nginx-plus-secret.md" >}} + +If using a name other than the default `nplus-license`, provide the name of this Secret when installing NGINX Gateway Fabric: + +{{}} + +{{%tab name="Helm"%}} + +Specify the Secret name using the `nginx.usage.secretName` helm value. + +{{% /tab %}} + +{{%tab name="Manifests"%}} + +Specify the Secret name in the `--usage-report-secret` command-line flag on the `nginx-gateway` container. + +You also need to define the proper volume mount to mount the Secret to the nginx container. If it doesn't already exist, add the following volume to the Deployment: + +```yaml +- name: nginx-plus-license + secret: + secretName: nplus-license +``` + +and the following volume mount to the `nginx` container: + +```yaml +- mountPath: /etc/nginx/license.jwt + name: nginx-plus-license + subPath: license.jwt +``` + +{{% /tab %}} + +{{}} + +### Reporting to NGINX Instance Manager {#nim} + +If you are deploying NGINX Gateway Fabric in an environment where you need to report to NGINX Instance Manager instead of the default licensing endpoint, a few extra steps may be required. + +First, you must specify the endpoint of your NGINX Instance Manager: + +{{}} + +{{%tab name="Helm"%}} + +Specify the endpoint using the `nginx.usage.endpoint` helm value. + +{{% /tab %}} + +{{%tab name="Manifests"%}} + +Specify the endpoint in the `--usage-report-endpoint` command-line flag on the `nginx-gateway` container. You also need to add the following line to the `mgmt` block of the `nginx-includes-bootstrap` ConfigMap: + +```text +usage_report endpoint=; +``` + +{{% /tab %}} + +{{}} + +#### CA and Client certificate/key {#nim-cert} + +To configure a CA cert and/or client certificate and key, a few extra steps are needed. + +First, you need to create two Secrets in the `nginx-gateway` namespace. The CA must live under the key `ca.crt`: + +```shell +kubectl -n nginx-gateway create secret generic nim-ca --from-file ca.crt +``` + +The client cert and key must be added to a TLS Secret: + +```shell +kubectl -n nginx-gateway create secret tls nim-client --cert /path/to/cert --key /path/to/key +``` + +{{}} + +{{%tab name="Helm"%}} + +Specify the CA Secret name using the `nginx.usage.caSecretName` helm value. Specify the client Secret name using the `nginx.usage.clientSSLSecretName` helm value. + +{{% /tab %}} + +{{%tab name="Manifests"%}} + +Specify the CA Secret name in the `--usage-report-ca-secret` command-line flag on the `nginx-gateway` container. Specify the client Secret name in the `--usage-report-client-ssl-secret` command-line flag on the `nginx-gateway` container. + +You also need to define the proper volume mount to mount the Secrets to the nginx container. Add the following volume to the Deployment: + +```yaml +- name: nginx-plus-usage-certs + projected: + sources: + - secret: + name: nim-ca + - secret: + name: nim-client +``` + +and the following volume mounts to the `nginx` container: + +```yaml +- mountPath: /etc/nginx/certs-bootstrap/ + name: nginx-plus-usage-certs +``` + +Finally, in the `nginx-includes-bootstrap` ConfigMap, add the following lines to the `mgmt` block: + +```text +ssl_trusted_certificate /etc/nginx/certs-bootstrap/ca.crt; +ssl_certificate /etc/nginx/certs-bootstrap/tls.crt; +ssl_certificate_key /etc/nginx/certs-bootstrap/tls.key; +``` + +{{% /tab %}} + +{{}} + +
+ +**Once these Secrets are created and configuration options are set, you can now [install NGINX Gateway Fabric]({{< relref "installation/installing-ngf" >}}).** + +## Installation flags for configuring usage reporting {#flags} + +When installing NGINX Gateway Fabric, the following flags can be specified to configure usage reporting to fit your needs: + +If using Helm, the `nginx.usage` values should be set as necessary: + +- `secretName` should be the name of the JWT Secret you created. By default this field is set to `nplus-license`. This field is required. +- `endpoint` is the endpoint to send the telemetry data to. This is optional, and by default is `product.connect.nginx.com`. +- `resolver` is the nameserver used to resolve the NGINX Plus usage reporting endpoint. This is optional and used with NGINX Instance Manager. +- `skipVerify` disables client verification of the NGINX Plus usage reporting server certificate. +- `caSecretName` is the name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). +- `clientSSLSecretName` is the name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + +If using manifests, the following command-line options should be set as necessary on the `nginx-gateway` container: + +- `--usage-report-secret` should be the name of the JWT Secret you created. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). By default this field is set to `nplus-license`. A [volume mount](#nginx-plus-secret) for this Secret is required for installation. +- `--usage-report-endpoint` is the endpoint to send the telemetry data to. This is optional, and by default is `product.connect.nginx.com`. Requires [extra configuration](#nim) if specified. +- `--usage-report-resolver` is the nameserver used to resolve the NGINX Plus usage reporting endpoint. This is optional and used with NGINX Instance Manager. +- `--usage-report-skip-verify` disables client verification of the NGINX Plus usage reporting server certificate. +- `--usage-report-ca-secret` is the name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). Requires [extra configuration](#nim-cert) if specified. +- `--usage-report-client-ssl-secret` is the name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). Requires [extra configuration](#nim-cert) if specified. + +## What’s reported and how it’s protected {#telemetry} + +NGINX Plus reports the following data every hour by default: + +- **NGINX version and status**: The version of NGINX Plus running on the instance. +- **Instance UUID**: A unique identifier for each NGINX Plus instance. +- **Traffic data**: + - **Bytes received from and sent to clients**: HTTP and stream traffic volume between clients and NGINX Plus. + - **Bytes received from and sent to upstreams**: HTTP and stream traffic volume between NGINX Plus and upstream servers. + - **Client connections**: The number of accepted client connections (HTTP and stream traffic). + - **Requests handled**: The total number of HTTP requests processed. +- **NGINX uptime**: The number of reloads and worker connections during uptime. +- **Usage report timestamps**: Start and end times for each usage report. +- **Kubernetes node details**: Information about Kubernetes nodes. + +### Security and privacy of reported data + +All communication between your NGINX Plus instances, NGINX Instance Manager, and F5’s licensing endpoint (`product.connect.nginx.com`) is protected using **SSL/TLS** encryption. + +Only **operational metrics** are reported — no **personally identifiable information (PII)** or **sensitive customer data** is transmitted. + +## Pulling an image for local use + +To pull an image for local use, use this command: + +```shell +docker login private-registry.nginx.com --username= --password=none +``` + +Replace the contents of `` with the contents of the JWT token itself. + +You can then pull the image: + +```shell +docker pull private-registry.nginx.com/nginx-gateway-fabric/nginx-plus:1.4.0 +``` + +Once you have successfully pulled the image, you can tag it as needed, then push it to a different container registry. + +## Alternative installation options + +There are alternative ways to get an NGINX Plus image for NGINX Gateway Fabric: + +- [Build the Gateway Fabric image]({{}}) describes how to use the source code with an NGINX Plus subscription certificate and key to build an image. diff --git a/site/content/installation/usage-reporting.md b/site/content/installation/usage-reporting.md deleted file mode 100644 index eb50658c91..0000000000 --- a/site/content/installation/usage-reporting.md +++ /dev/null @@ -1,185 +0,0 @@ ---- -title: "Enable Usage Reporting for NGINX Plus" -weight: 1000 -toc: true -docs: "DOCS-000" ---- - -## Overview - -This document describes how to enable Usage Reporting for NGINX Gateway Fabric and how to view the usage data through the API. - -Usage Reporting connects to the NGINX Instance Manager and reports the number of Nodes and NGINX Gateway Fabric Pods in the cluster. - -To use Usage Reporting, you must have access to [NGINX Instance Manager](https://www.f5.com/products/nginx/instance-manager). Usage Reporting is a requirement of the new Flexible Consumption Program for NGINX Gateway Fabric, used to calculate costs. **This only applies if using NGINX Plus as the data plane.** Usage is reported every 24 hours. - -## Requirements - -Usage Reporting needs to be configured when deploying NGINX Gateway Fabric. - -To enable Usage Reporting, you must have the following: - -- NGINX Gateway Fabric 1.2.0 or later -- [NGINX Instance Manager 2.11](https://docs.nginx.com/nginx-management-suite) or later - -In addition to the software requirements, you will need: - -- Access to an NGINX Instance Manager username and password for basic authentication. You will also need the URL of your NGINX Instance Manager system. The Usage Reporting user account must have access to the `/api/platform/v1/k8s-usage` endpoint. -- Access to the Kubernetes cluster where NGINX Gateway Fabric is deployed, with the ability to deploy a Kubernetes Secret. - -## Adding a User Account to NGINX Instance Manager - -1. Create a role following the steps in the [Create Roles](https://docs.nginx.com/nginx-management-suite/admin-guides/rbac/create-roles/) section of the NGINX Instance Manager documentation. Select these permissions in step 6 for the role: - -- Module: Instance Manager -- Feature: NGINX Plus Usage -- Access: CRUD - -1. Create a user account following the steps in the [Create New Users](https://docs.nginx.com/nginx-management-suite/admin-guides/authentication/basic-authentication/#create-users) section of the NGINX Instance Manager documentation. In step 6, assign the user to the role created above. Note that currently only "Basic Auth" authentication is supported for Usage Reporting purposes. - -## Enabling Usage Reporting in NGINX Gateway Fabric - -### Adding Credentials to a Kubernetes Secret - -To make the credentials available to NGINX Gateway Fabric to connect to the NGINX Instance Manager, we need to create a Kubernetes Secret. - -1. The username and password should be base64 encoded and stored in the Secret. In the following example, the username is `foo` and the password is `bar`. Run a similar command to generate the base64 encoded strings of your username and password: - - ```shell - echo -n 'foo' | base64 - # Zm9v - echo -n 'bar' | base64 - # YmFy - ``` - -1. Create the following Secret in your Kubernetes cluster, replacing the username and password with your generated strings. You can rename the Secret if desired, and create it in any Namespace you want. - - ```yaml - apiVersion: v1 - kind: Secret - metadata: - name: ngf-usage-auth - namespace: nginx-gateway - type: kubernetes.io/basic-auth - data: - username: Zm9v # base64 representation of 'foo' obtained in step 1 - password: YmFy # base64 representation of 'bar' obtained in step 1 - ``` - - If you need to update the basic-auth credentials at any time, update the `username` and `password` fields and apply the changes. NGINX Gateway Fabric will automatically detect the changes and use the new username and password without redeployment. - -### Install NGINX Gateway Fabric with Usage Reporting enabled - -When installing NGINX Gateway Fabric, a few configuration options need to be specified in order to enable Usage Reporting. You should follow the normal [installation](https://docs.nginx.com/nginx-gateway-fabric/installation/) steps using your preferred method, but ensure you include the following options: - -If using Helm, the `nginx.usage` values should be set as necessary: - -- `secretName` should be the `namespace/name` of the credentials Secret you created. Using our example, it would be `nginx-gateway/ngf-usage-auth`. This field is required. -- `serverURL` is the base server URL of the NGINX Instance Manager. This field is required. -- `clusterName` is an optional display name in the API for the usage data object. - -If using manifests, the following command-line options should be set as necessary on the `nginx-gateway` container: - -- `--usage-report-secret` should be the `namespace/name` of the credentials Secret you created. Using our example, it would be `nginx-gateway/ngf-usage-auth`. This field is required. -- `--usage-report-server-url` is the base server URL of the NGINX Instance Manager. This field is required. -- `--usage-report-cluster-name` is an optional display name in the API for the usage data object. - -Your NGINX Gateway Fabric Pods should also have one of the following labels: - -- `app.kubernetes.io/name=nginx-gateway` -- `app.kubernetes.io/name=nginx-gateway-fabric` - -{{< note >}}The default installation of NGINX Gateway Fabric already includes at least one of these labels.{{< /note >}} - -## Viewing Usage Data from the NGINX Instance Manager API - -NGINX Gateway Fabric sends the number of its instances and nodes in the cluster to NGINX Instance Manager every 24 hours. To view the usage data, query the NGINX Instance Manager API. The usage data is available at the following endpoint (replace `nim.example.com` with your server URL, and set the proper credentials in the `--user` field): - -```shell -curl --user "foo:bar" https://nim.example.com/api/platform/v1/k8s-usage -``` - -```json -{ - "items": [ - { - "max_node_count": 5, - "metadata": { - "createTime": "2023-01-27T09:12:33.001Z", - "displayName": "my-cluster", - "monthReturned": "May", - "uid": "d290f1ee-6c54-4b01-90e6-d701748f0851", - "updateTime": "2023-01-29T10:12:33.001Z" - }, - "node_count": 4, - "pod_details": { - "current_pod_counts": { - "dos_count": 0, - "pod_count": 15, - "waf_count": 0 - }, - "max_pod_counts": { - "max_dos_count": 0, - "max_pod_count": 25, - "max_waf_count": 0 - } - } - }, - { - "max_node_count": 3, - "metadata": { - "createTime": "2023-01-25T09:12:33.001Z", - "displayName": "my-cluster2", - "monthReturned": "May", - "uid": "12tgb8ug-g8ik-bs7h-gj3j-hjitk672946hb", - "updateTime": "2023-01-26T10:12:33.001Z" - }, - "node_count": 3, - "pod_details": { - "current_pod_counts": { - "dos_count": 0, - "pod_count": 5, - "waf_count": 0 - }, - "max_pod_counts": { - "max_dos_count": 0, - "max_pod_count": 15, - "max_waf_count": 0 - } - } - } - ] -} -``` - -You can also query the usage data for a specific cluster by specifying the cluster uid in the endpoint, for example: - -```shell -curl --user "foo:bar" https://nim.example.com/api/platform/v1/k8s-usage/d290f1ee-6c54-4b01-90e6-d701748f0851 -``` - -```json -{ - "max_node_count": 5, - "metadata": { - "createTime": "2023-01-27T09:12:33.001Z", - "displayName": "my-cluster", - "monthReturned": "May", - "uid": "d290f1ee-6c54-4b01-90e6-d701748f0851", - "updateTime": "2023-01-29T10:12:33.001Z" - }, - "node_count": 4, - "pod_details": { - "current_pod_counts": { - "dos_count": 0, - "pod_count": 15, - "waf_count": 0 - }, - "max_pod_counts": { - "max_dos_count": 0, - "max_pod_count": 25, - "max_waf_count": 0 - } - } -} -``` diff --git a/site/content/reference/cli-help.md b/site/content/reference/cli-help.md index 5122212a89..aedcc0e74c 100644 --- a/site/content/reference/cli-help.md +++ b/site/content/reference/cli-help.md @@ -22,6 +22,7 @@ This command configures NGINX for a single NGINX Gateway Fabric resource. ### Flags {{< bootstrap-table "table table-bordered table-striped table-responsive" >}} + | Name | Type | Description | |-------------------------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | _gateway-ctlr-name_ | _string_ | The name of the Gateway controller. The controller name must be in the form: `DOMAIN/PATH`. The controller's domain is `gateway.nginx.org`. | @@ -40,11 +41,14 @@ This command configures NGINX for a single NGINX Gateway Fabric resource. | _leader-election-disable_ | _bool_ | Disable leader election, which is used to avoid multiple replicas of the NGINX Gateway Fabric reporting the status of the Gateway API resources. If disabled, all replicas of NGINX Gateway Fabric will update the statuses of the Gateway API resources (Default: `false`). | | _leader-election-lock-name_ | _string_ | The name of the leader election lock. A lease object with this name will be created in the same namespace as the controller (Default: `"nginx-gateway-leader-election-lock"`). | | _product-telemetry-disable_ | _bool_ | Disable the collection of product telemetry (Default: `false`). | -| _usage-report-secret_ | _string_ | The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting. | -| _usage-report-server-url_ | _string_ | The base server URL of the NGINX Plus usage reporting server. | -| _usage-report-cluster-name_ | _string_ | The display name of the Kubernetes cluster in the NGINX Plus usage reporting server. | +| _usage-report-secret_ | _string_ | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway) | +| _usage-report-endpoint_ | _string_ | The endpoint of the NGINX Plus usage reporting server. | +| _usage-report-resolver_ | _string_ | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | | _usage-report-skip-verify_ | _bool_ | Disable client verification of the NGINX Plus usage reporting server certificate. | +| _usage-report-ca-secret_ | _string_ | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway) | +| _usage-report-client-ssl-secret_ | _string_ | TThe name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway) | | _snippets-filters_ | _bool_ | Enable SnippetsFilters feature. SnippetsFilters allow inserting NGINX configuration into the generated NGINX config for HTTPRoute and GRPCRoute resources. | + {{% /bootstrap-table %}} ## Sleep @@ -58,7 +62,9 @@ _Usage_: ``` {{< bootstrap-table "table table-bordered table-striped table-responsive" >}} + | Name | Type | Description | | -------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------- | | duration | `time.Duration` | Set the duration of sleep. Must be parsable by [`time.ParseDuration`](https://pkg.go.dev/time#ParseDuration). (default `30s`) | + {{% /bootstrap-table %}} diff --git a/tests/Makefile b/tests/Makefile index fdb25a87d5..529e052322 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -127,9 +127,10 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \ --label-filter "nfr" $(GINKGO_FLAGS) --timeout 5h ./suite -- --gateway-api-version=$(GW_API_VERSION) \ --gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \ - --plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ + --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ --pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \ - --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) + --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --plus-enabled=$(PLUS_ENABLED) \ + --plus-license-file-name=$(PLUS_LICENSE_FILE) .PHONY: test test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster @@ -139,9 +140,10 @@ test: build-crossplane-image ## Runs the functional tests on your kind k8s clust --label-filter "functional" $(GINKGO_FLAGS) ./suite -- \ --gateway-api-version=$(GW_API_VERSION) --gateway-api-prev-version=$(GW_API_PREV_VERSION) \ --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \ - --plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ + --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ --pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \ - --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --cluster-name=$(CLUSTER_NAME) + --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --cluster-name=$(CLUSTER_NAME) --plus-enabled=$(PLUS_ENABLED) \ + --plus-license-file-name=$(PLUS_LICENSE_FILE) .PHONY: test-with-plus test-with-plus: PLUS_ENABLED=true diff --git a/tests/framework/ngf.go b/tests/framework/ngf.go index 8cf0ecd0f7..6c3008d882 100644 --- a/tests/framework/ngf.go +++ b/tests/framework/ngf.go @@ -3,6 +3,7 @@ package framework import ( "context" "fmt" + "os" "os/exec" "path/filepath" "strings" @@ -16,7 +17,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const gwInstallBasePath = "https://github.com/kubernetes-sigs/gateway-api/releases/download" +const ( + gwInstallBasePath = "https://github.com/kubernetes-sigs/gateway-api/releases/download" + PlusSecretName = "nplus-license" +) // InstallationConfig contains the configuration for the NGF installation. type InstallationConfig struct { @@ -81,6 +85,43 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { return exec.Command("helm", fullArgs...).CombinedOutput() } +// CreateLicenseSecret creates the NGINX Plus JWT secret. +func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) error { + conf, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("error reading file %q: %w", filename, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeoutConfig().CreateTimeout) + defer cancel() + + ns := &core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + + if err := k8sClient.Create(ctx, ns); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("error creating namespace: %w", err) + } + + secret := &core.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: PlusSecretName, + Namespace: namespace, + }, + Data: map[string][]byte{ + "license.jwt": conf, + }, + } + + if err := k8sClient.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("error creating secret: %w", err) + } + + return nil +} + // UpgradeNGF upgrades NGF. CRD upgrades assume the chart is local. func UpgradeNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { crdPath := filepath.Join(cfg.ChartPath, "crds") + "/" diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 0e66c5f99b..f837ba9ede 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -3,6 +3,7 @@ set -o pipefail SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +REPO_DIR=$(dirname $(dirname "$SCRIPT_DIR")) source scripts/vars.env @@ -55,6 +56,8 @@ git clone https://github.com/${NGF_REPO}/nginx-gateway-fabric.git EOF" -- -t fi +gcloud compute scp --quiet --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${REPO_DIR}"/license.jwt username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/ + gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ --command="bash -i <