Code review; watch for secret updates

Author: sjberman

charts/nginx-gateway-fabric/README.md

@@ -268,10 +268,10 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.image.tag` |  | string | `"edge"` |
 | `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` |
 | `nginx.plus` | Is NGINX Plus image being used | bool | `false` |
-| `nginx.usage.caSecretName` | The name of the Secret containing the CA cert for verifying the NGINX Plus usage reporting server. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
-| `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client cert/key for communicating with the NGINX Plus usage reporting server. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
+| `nginx.usage.caSecretName` | The name of the Secret containing the CA certificate for verifying the NGINX Plus usage reporting server. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
+| `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate/key for communicating with the NGINX Plus usage reporting server. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` |
-| `nginx.usage.resolver` | The resolver domain name or IP address with optional port for resolving the endpoint. | string | `""` |
+| `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` |
 | `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` |
 | `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` |
 | `nginxGateway.config.logging.level` | Log level. | string | `"info"` |

charts/nginx-gateway-fabric/templates/configmap.yaml

@@ -15,19 +15,18 @@ data:
   {{- if .Values.nginx.plus }}
   mgmt.conf: |
     mgmt {
-        license_token /etc/nginx/license/license.jwt;
         {{- if .Values.nginx.usage.endpoint }}
         usage_report endpoint={{ .Values.nginx.usage.endpoint }};
         {{- end }}
         {{- if .Values.nginx.usage.skipVerify }}
         ssl_verify off;
         {{- end }}
         {{- if .Values.nginx.usage.caSecretName }}
-        ssl_trusted_certificate /etc/nginx/usage-certs/ca/ca.crt;
+        ssl_trusted_certificate /etc/nginx/certs-bootstrap/ca.crt;
         {{- end }}
         {{- if .Values.nginx.usage.clientSSLSecretName }}
-        ssl_certificate        /etc/nginx/usage-certs/client/tls.crt;
-        ssl_certificate_key    /etc/nginx/usage-certs/client/tls.key;
+        ssl_certificate        /etc/nginx/certs-bootstrap/tls.crt;
+        ssl_certificate_key    /etc/nginx/certs-bootstrap/tls.key;
         {{- end }}
         enforce_initial_report off;
     }

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -229,15 +229,12 @@ spec:
         {{- if .Values.nginx.plus }}
           {{- if .Values.nginx.usage.secretName }}
         - name: nginx-plus-license
-          mountPath: /etc/nginx/license
+          mountPath: /etc/nginx/license.jwt
+          subPath: license.jwt
           {{- end }}
-          {{- if .Values.nginx.usage.clientSSLSecretName }}
-        - name: usage-client-ssl-secret
-          mountPath: /etc/nginx/usage-certs/client
-          {{- end }}
-          {{- if .Values.nginx.usage.caSecretName }}
-        - name: usage-ca-secret
-          mountPath: /etc/nginx/usage-certs/ca
+          {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }}
+        - name: nginx-plus-usage-certs
+          mountPath: /etc/nginx/certs-bootstrap/
           {{- end }}
         {{- end }}
         {{- with .Values.nginx.extraVolumeMounts -}}
@@ -294,15 +291,18 @@ spec:
         secret:
           secretName: {{ .Values.nginx.usage.secretName }}
         {{- end }}
-        {{- if .Values.nginx.usage.clientSSLSecretName }}
-      - name: usage-client-ssl-secret
-        secret:
-          secretName: {{ .Values.nginx.usage.clientSSLSecretName }}
-        {{- end }}
-        {{- if .Values.nginx.usage.caSecretName }}
-      - name: usage-ca-secret
-        secret:
-          secretName: {{ .Values.nginx.usage.caSecretName }}
+        {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }}
+      - name: nginx-plus-usage-certs
+        projected:
+          sources:
+          {{- if .Values.nginx.usage.caSecretName }}
+          - secret:
+              name: {{ .Values.nginx.usage.caSecretName }}
+          {{- end }}
+          {{- if .Values.nginx.usage.clientSSLSecretName }}
+          - secret:
+              name: {{ .Values.nginx.usage.clientSSLSecretName }}
+          {{- end }}
         {{- end }}
       {{- end }}
       {{- with .Values.extraVolumes -}}

charts/nginx-gateway-fabric/templates/scc.yaml

@@ -33,6 +33,7 @@ volumes:
 - emptyDir
 - secret
 - configMap
+- projected
 users:
 - {{ printf "system:serviceaccount:%s:%s" .Release.Namespace (include "nginx-gateway.serviceAccountName" .) }}
 allowedCapabilities:

charts/nginx-gateway-fabric/values.schema.json

@@ -264,14 +264,14 @@
           "properties": {
             "caSecretName": {
               "default": "",
-              "description": "The name of the Secret containing the CA cert for verifying the NGINX Plus usage reporting\nserver. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).",
+              "description": "The name of the Secret containing the CA certificate for verifying the NGINX Plus usage reporting\nserver. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).",
               "required": [],
               "title": "caSecretName",
               "type": "string"
             },
             "clientSSLSecretName": {
               "default": "",
-              "description": "The name of the Secret containing the client cert/key for communicating with the NGINX Plus usage reporting\nserver. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).",
+              "description": "The name of the Secret containing the client certificate/key for communicating with the NGINX Plus usage reporting\nserver. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).",
               "required": [],
               "title": "clientSSLSecretName",
               "type": "string"
@@ -285,7 +285,7 @@
             },
             "resolver": {
               "default": "",
-              "description": "The resolver domain name or IP address with optional port for resolving the endpoint.",
+              "description": "The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager.",
               "required": [],
               "title": "resolver",
               "type": "string"

charts/nginx-gateway-fabric/values.yaml

@@ -143,17 +143,17 @@ nginx:
     # -- The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com
     endpoint: ""
 
-    # -- The resolver domain name or IP address with optional port for resolving the endpoint.
+    # -- The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager.
     resolver: ""
 
     # -- Disable client verification of the NGINX Plus usage reporting server certificate.
     skipVerify: false
 
-    # -- The name of the Secret containing the CA cert for verifying the NGINX Plus usage reporting
+    # -- The name of the Secret containing the CA certificate for verifying the NGINX Plus usage reporting
     # server. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).
     caSecretName: ""
 
-    # -- The name of the Secret containing the client cert/key for communicating with the NGINX Plus usage reporting
+    # -- The name of the Secret containing the client certificate/key for communicating with the NGINX Plus usage reporting
     # server. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).
     clientSSLSecretName: ""
 

cmd/gateway/commands.go

@@ -392,6 +392,8 @@ func createStaticModeCommand() *cobra.Command {
 			"that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).",
 	)
 
+	cmd.MarkFlagsRequiredTogether(plusFlag, usageReportSecretFlag)
+
 	cmd.Flags().Var(
 		&usageReportEndpoint,
 		usageReportEndpointFlag,
@@ -401,7 +403,7 @@ func createStaticModeCommand() *cobra.Command {
 	cmd.Flags().Var(
 		&usageReportResolver,
 		usageReportResolverFlag,
-		"The nameserver used to resolve the NGINX Plus usage reporting server name.",
+		"The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager.",
 	)
 
 	cmd.Flags().BoolVar(

cmd/gateway/commands_test.go

@@ -153,6 +153,7 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
 				"--nginx-plus",
 				"--usage-report-secret=my-secret",
 				"--usage-report-endpoint=example.com",
+				"--usage-report-resolver=resolver.com",
 				"--usage-report-ca-secret=ca-secret",
 				"--usage-report-client-ssl-secret=client-secret",
 				"--snippets-filters",
@@ -344,6 +345,23 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
 			expectedErrPrefix: `invalid argument "$*(invalid)" for "--usage-report-endpoint" flag: ` +
 				`"$*(invalid)" must be a domain name or IP address with optional port`,
 		},
+		{
+			name: "usage-report-resolver is set to empty string",
+			args: []string{
+				"--usage-report-resolver=",
+			},
+			wantErr:           true,
+			expectedErrPrefix: `invalid argument "" for "--usage-report-resolver" flag: must be set`,
+		},
+		{
+			name: "usage-report-resolveris an invalid endpoint",
+			args: []string{
+				"--usage-report-resolver=$*(invalid)",
+			},
+			wantErr: true,
+			expectedErrPrefix: `invalid argument "$*(invalid)" for "--usage-report-resolver" flag: ` +
+				`"$*(invalid)" must be a domain name or IP address with optional port`,
+		},
 		{
 			name: "usage-report-ca-secret is set to empty string",
 			args: []string{

deploy/experimental-nginx-plus/deploy.yaml

@@ -161,7 +161,6 @@ data:
     error_log stderr info;
   mgmt.conf: |
     mgmt {
-        license_token /etc/nginx/license/license.jwt;
         enforce_initial_report off;
     }
 kind: ConfigMap
@@ -323,8 +322,9 @@ spec:
           name: nginx-lib
         - mountPath: /etc/nginx/includes
           name: nginx-includes
-        - mountPath: /etc/nginx/license
+        - mountPath: /etc/nginx/license.jwt
           name: nginx-plus-license
+          subPath: license.jwt
       initContainers:
       - command:
         - /usr/bin/gateway

deploy/nginx-plus/deploy.yaml

@@ -156,7 +156,6 @@ data:
     error_log stderr info;
   mgmt.conf: |
     mgmt {
-        license_token /etc/nginx/license/license.jwt;
         enforce_initial_report off;
     }
 kind: ConfigMap
@@ -317,8 +316,9 @@ spec:
           name: nginx-lib
         - mountPath: /etc/nginx/includes
           name: nginx-includes
-        - mountPath: /etc/nginx/license
+        - mountPath: /etc/nginx/license.jwt
           name: nginx-plus-license
+          subPath: license.jwt
       initContainers:
       - command:
         - /usr/bin/gateway

deploy/openshift/deploy.yaml

@@ -429,3 +429,4 @@ volumes:
 - emptyDir
 - secret
 - configMap
+- projected

deploy/snippets-filters-nginx-plus/deploy.yaml

@@ -158,7 +158,6 @@ data:
     error_log stderr info;
   mgmt.conf: |
     mgmt {
-        license_token /etc/nginx/license/license.jwt;
         enforce_initial_report off;
     }
 kind: ConfigMap
@@ -320,8 +319,9 @@ spec:
           name: nginx-lib
         - mountPath: /etc/nginx/includes
           name: nginx-includes
-        - mountPath: /etc/nginx/license
+        - mountPath: /etc/nginx/license.jwt
           name: nginx-plus-license
+          subPath: license.jwt
       initContainers:
       - command:
         - /usr/bin/gateway

internal/mode/static/config/config.go

@@ -106,15 +106,15 @@ type ProductTelemetryConfig struct {
 type UsageReportConfig struct {
 	// SecretName is the name of the Secret containing the server credentials.
 	SecretName string
-	// ClientSSLSecretName is the name of the Secret containing client cert/key.
+	// ClientSSLSecretName is the name of the Secret containing client certificate/key.
 	ClientSSLSecretName string
 	// CASecretName is the name of the Secret containing the CA certificate.
 	CASecretName string
 	// Endpoint is the endpoint of the reporting server.
 	Endpoint string
 	// Resolver is the nameserver for resolving the Endpoint.
 	Resolver string
-	// SkipVerify controls whether the nginx verifies the server cert.
+	// SkipVerify controls whether the nginx verifies the server certificate.
 	SkipVerify bool
 }
 

internal/mode/static/handler.go

@@ -29,6 +29,7 @@ import (
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/status"
+	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/telemetry"
 )
 
 type handlerMetricsCollector interface {
@@ -51,8 +52,10 @@ type eventHandlerConfig struct {
 	serviceResolver resolver.ServiceResolver
 	// generator is the nginx config generator.
 	generator ngxConfig.Generator
-	// k8sClient is a Kubernetes API client
+	// k8sClient is a Kubernetes API client.
 	k8sClient client.Client
+	// k8sReader is a Kubernets API reader.
+	k8sReader client.Reader
 	// logLevelSetter is used to update the logging level.
 	logLevelSetter logLevelSetter
 	// eventRecorder records events for Kubernetes resources.
@@ -67,6 +70,8 @@ type eventHandlerConfig struct {
 	gatewayCtlrName string
 	// updateGatewayClassStatus enables updating the status of the GatewayClass resource.
 	updateGatewayClassStatus bool
+	// plus is whether or not we are running NGINX Plus.
+	plus bool
 }
 
 const (
@@ -168,6 +173,9 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log
 	case state.EndpointsOnlyChange:
 		h.version++
 		cfg := dataplane.BuildConfiguration(ctx, gr, h.cfg.serviceResolver, h.version)
+		if err := h.setDeploymentCtx(ctx, &cfg); err != nil {
+			logger.Error(err, "error setting deployment context for usage reporting")
+		}
 
 		h.setLatestConfiguration(&cfg)
 
@@ -179,6 +187,9 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log
 	case state.ClusterStateChange:
 		h.version++
 		cfg := dataplane.BuildConfiguration(ctx, gr, h.cfg.serviceResolver, h.version)
+		if err := h.setDeploymentCtx(ctx, &cfg); err != nil {
+			logger.Error(err, "error setting deployment context for usage reporting")
+		}
 
 		h.setLatestConfiguration(&cfg)
 
@@ -485,6 +496,42 @@ func getGatewayAddresses(
 	return gwAddresses, nil
 }
 
+// setDeploymentCtx sets the deployment context metadata for nginx plus reporting.
+func (h *eventHandlerImpl) setDeploymentCtx(ctx context.Context, cfg *dataplane.Configuration) error {
+	if !h.cfg.plus {
+		return nil
+	}
+
+	podNSName := types.NamespacedName{
+		Name:      h.cfg.gatewayPodConfig.Name,
+		Namespace: h.cfg.gatewayPodConfig.Namespace,
+	}
+
+	clusterInfo, err := telemetry.CollectClusterInformation(ctx, h.cfg.k8sReader)
+	if err != nil {
+		return fmt.Errorf("error getting cluster information")
+	}
+
+	replicaSet, err := telemetry.GetPodReplicaSet(ctx, h.cfg.k8sReader, podNSName)
+	if err != nil {
+		return fmt.Errorf("failed to get replica set for pod %v: %w", podNSName, err)
+	}
+
+	deploymentID, err := telemetry.GetDeploymentID(replicaSet)
+	if err != nil {
+		return fmt.Errorf("failed to get NGF deploymentID: %w", err)
+	}
+
+	cfg.DeploymentContext = dataplane.DeploymentContext{
+		Integration:      "ngf",
+		ClusterID:        clusterInfo.ClusterID,
+		ClusterNodeCount: clusterInfo.NodeCount,
+		InstallationID:   deploymentID,
+	}
+
+	return nil
+}
+
 // GetLatestConfiguration gets the latest configuration.
 func (h *eventHandlerImpl) GetLatestConfiguration() *dataplane.Configuration {
 	h.lock.Lock()