Collect telemetry for control data plane split

Author: bjee19

internal/mode/static/telemetry/collector.go

@@ -12,11 +12,13 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	k8sversion "k8s.io/apimachinery/pkg/util/version"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	ngfAPI "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1"
+	"github.com/nginx/nginx-gateway-fabric/internal/framework/controller"
 	"github.com/nginx/nginx-gateway-fabric/internal/framework/kinds"
 	"github.com/nginx/nginx-gateway-fabric/internal/mode/static/config"
 	"github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/dataplane"
@@ -60,8 +62,10 @@ type Data struct {
 	// then lastly by directive string.
 	SnippetsFiltersDirectivesCount []int64
 	NGFResourceCounts              // embedding is required by the generator.
-	// NGFReplicaCount is the number of replicas of the NGF Pod.
-	NGFReplicaCount int64
+	// NginxPodCount is the total number of Nginx data plane Pods.
+	NginxPodCount int64
+	// ControlPlanePodCount is the total number of NGF control plane Pods.
+	ControlPlanePodCount int64
 }
 
 // NGFResourceCounts stores the counts of all relevant resources that NGF processes and generates configuration from.
@@ -99,6 +103,8 @@ type NGFResourceCounts struct {
 	SnippetsFilterCount int64
 	// UpstreamSettingsPolicyCount is the number of UpstreamSettingsPolicies.
 	UpstreamSettingsPolicyCount int64
+	// GatewayAttachedNpCount is the total number of NginxProxy resources that are attached to a Gateway.
+	GatewayAttachedNpCount int64
 }
 
 // DataCollectorConfig holds configuration parameters for DataCollectorImpl.
@@ -152,18 +158,17 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) {
 		return Data{}, fmt.Errorf("failed to get replica set for pod %v: %w", c.cfg.PodNSName, err)
 	}
 
-	replicaCount, err := getReplicas(replicaSet)
-	if err != nil {
-		return Data{}, fmt.Errorf("failed to collect NGF replica count: %w", err)
-	}
-
 	deploymentID, err := getDeploymentID(replicaSet)
 	if err != nil {
 		return Data{}, fmt.Errorf("failed to get NGF deploymentID: %w", err)
 	}
 
 	snippetsFiltersDirectives, snippetsFiltersDirectivesCount := collectSnippetsFilterDirectives(g)
 
+	nginxPodCount := getNginxPodCount(g)
+
+	controlPlanePodCount := getControlPlanePodCount(ctx, c.cfg.K8sClientReader)
+
 	data := Data{
 		Data: tel.Data{
 			ProjectName:         "NGF",
@@ -179,9 +184,10 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) {
 		ImageSource:                    c.cfg.ImageSource,
 		FlagNames:                      c.cfg.Flags.Names,
 		FlagValues:                     c.cfg.Flags.Values,
-		NGFReplicaCount:                int64(replicaCount),
 		SnippetsFiltersDirectives:      snippetsFiltersDirectives,
 		SnippetsFiltersDirectivesCount: snippetsFiltersDirectivesCount,
+		NginxPodCount:                  nginxPodCount,
+		ControlPlanePodCount:           controlPlanePodCount,
 	}
 
 	return data, nil
@@ -241,6 +247,18 @@ func collectGraphResourceCount(
 	ngfResourceCounts.NginxProxyCount = int64(len(g.ReferencedNginxProxies))
 	ngfResourceCounts.SnippetsFilterCount = int64(len(g.SnippetsFilters))
 
+	var gatewayAttachedNPCount int64
+	if g.GatewayClass != nil && g.GatewayClass.NginxProxy != nil {
+		gatewayClassNP := g.GatewayClass.NginxProxy
+		for _, np := range g.ReferencedNginxProxies {
+			if np != gatewayClassNP {
+				gatewayAttachedNPCount++
+			}
+		}
+	}
+
+	ngfResourceCounts.GatewayAttachedNpCount = gatewayAttachedNPCount
+
 	return ngfResourceCounts
 }
 
@@ -309,14 +327,6 @@ func getPodReplicaSet(
 	return &replicaSet, nil
 }
 
-func getReplicas(replicaSet *appsv1.ReplicaSet) (int, error) {
-	if replicaSet.Spec.Replicas == nil {
-		return 0, errors.New("replica set replicas was nil")
-	}
-
-	return int(*replicaSet.Spec.Replicas), nil
-}
-
 // getDeploymentID gets the deployment ID of the provided ReplicaSet.
 func getDeploymentID(replicaSet *appsv1.ReplicaSet) (string, error) {
 	replicaOwnerRefs := replicaSet.GetOwnerReferences()
@@ -495,3 +505,31 @@ func parseDirectiveContextMapIntoLists(directiveContextMap map[sfDirectiveContex
 
 	return directiveContextList, countList
 }
+
+func getNginxPodCount(g *graph.Graph) int64 {
+	var count int64
+	for _, gateway := range g.Gateways {
+		np := gateway.EffectiveNginxProxy
+		if np != nil &&
+			np.Kubernetes != nil &&
+			np.Kubernetes.Deployment != nil &&
+			np.Kubernetes.Deployment.Replicas != nil {
+			count += int64(*np.Kubernetes.Deployment.Replicas)
+		}
+	}
+
+	return count
+}
+
+func getControlPlanePodCount(ctx context.Context, k8sClient client.Reader) int64 {
+	var podList v1.PodList
+	opts := &client.ListOptions{
+		LabelSelector: labels.SelectorFromSet(labels.Set{controller.AppNameLabel: "nginx-gateway-fabric"}),
+	}
+
+	if err := k8sClient.List(ctx, &podList, opts); err != nil {
+		return 0
+	}
+
+	return int64(len(podList.Items))
+}

internal/mode/static/telemetry/collector_test.go

@@ -18,6 +18,9 @@ import (
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	ngfAPI "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1"
+	"github.com/nginx/nginx-gateway-fabric/apis/v1alpha2"
+	"github.com/nginx/nginx-gateway-fabric/internal/framework/controller"
+	"github.com/nginx/nginx-gateway-fabric/internal/framework/helpers"
 	"github.com/nginx/nginx-gateway-fabric/internal/framework/kinds"
 	"github.com/nginx/nginx-gateway-fabric/internal/framework/kubernetes/kubernetesfakes"
 	"github.com/nginx/nginx-gateway-fabric/internal/mode/static/config"
@@ -35,9 +38,7 @@ type listCallsFunc = func(
 ) error
 
 func createListCallsFunc(objects ...client.ObjectList) listCallsFunc {
-	return func(_ context.Context, object client.ObjectList, option ...client.ListOption) error {
-		Expect(option).To(BeEmpty())
-
+	return func(_ context.Context, object client.ObjectList, _ ...client.ListOption) error {
 		for _, obj := range objects {
 			if reflect.TypeOf(obj) == reflect.TypeOf(object) {
 				reflect.ValueOf(object).Elem().Set(reflect.ValueOf(obj).Elem())
@@ -87,6 +88,7 @@ var _ = Describe("Collector", Ordered, func() {
 		baseListCalls           listCallsFunc
 		flags                   config.Flags
 		nodeList                *v1.NodeList
+		podList                 *v1.PodList
 	)
 
 	BeforeAll(func() {
@@ -155,6 +157,17 @@ var _ = Describe("Collector", Ordered, func() {
 				},
 			},
 		}
+
+		podList = &v1.PodList{
+			Items: []v1.Pod{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:   "ngf-pod-1",
+						Labels: map[string]string{controller.AppNameLabel: "nginx-gateway-fabric"},
+					},
+				},
+			},
+		}
 	})
 
 	BeforeEach(func() {
@@ -170,7 +183,7 @@ var _ = Describe("Collector", Ordered, func() {
 				ClusterNodeCount:    1,
 			},
 			NGFResourceCounts:              telemetry.NGFResourceCounts{},
-			NGFReplicaCount:                1,
+			ControlPlanePodCount:           1,
 			ImageSource:                    "local",
 			FlagNames:                      flags.Names,
 			FlagValues:                     flags.Values,
@@ -198,7 +211,7 @@ var _ = Describe("Collector", Ordered, func() {
 		baseGetCalls = createGetCallsFunc(ngfPod, ngfReplicaSet, kubeNamespace)
 		k8sClientReader.GetCalls(baseGetCalls)
 
-		baseListCalls = createListCallsFunc(nodeList)
+		baseListCalls = createListCallsFunc(nodeList, podList)
 		k8sClientReader.ListCalls(baseListCalls)
 	})
 
@@ -260,7 +273,24 @@ var _ = Describe("Collector", Ordered, func() {
 					},
 				}
 
-				k8sClientReader.ListCalls(createListCallsFunc(nodes))
+				podList := &v1.PodList{
+					Items: []v1.Pod{
+						{
+							ObjectMeta: metav1.ObjectMeta{
+								Name:   "ngf-pod-1",
+								Labels: map[string]string{controller.AppNameLabel: "nginx-gateway-fabric"},
+							},
+						},
+						{
+							ObjectMeta: metav1.ObjectMeta{
+								Name:   "ngf-pod-2",
+								Labels: map[string]string{controller.AppNameLabel: "nginx-gateway-fabric"},
+							},
+						},
+					},
+				}
+
+				k8sClientReader.ListCalls(createListCallsFunc(nodes, podList))
 
 				secret1 := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret1"}}
 				secret2 := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret2"}}
@@ -270,11 +300,33 @@ var _ = Describe("Collector", Ordered, func() {
 				svc2 := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "svc2"}}
 				nilsvc := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nilsvc"}}
 
+				gcNP := graph.NginxProxy{
+					Source:  nil,
+					ErrMsgs: nil,
+					Valid:   false,
+				}
+
 				graph := &graph.Graph{
-					GatewayClass: &graph.GatewayClass{},
+					GatewayClass: &graph.GatewayClass{NginxProxy: &gcNP},
 					Gateways: map[types.NamespacedName]*graph.Gateway{
-						{Name: "gateway1"}: {},
-						{Name: "gateway2"}: {},
+						{Name: "gateway1"}: {
+							EffectiveNginxProxy: &graph.EffectiveNginxProxy{
+								Kubernetes: &v1alpha2.KubernetesSpec{
+									Deployment: &v1alpha2.DeploymentSpec{
+										Replicas: helpers.GetPointer(int32(1)),
+									},
+								},
+							},
+						},
+						{Name: "gateway2"}: {
+							EffectiveNginxProxy: &graph.EffectiveNginxProxy{
+								Kubernetes: &v1alpha2.KubernetesSpec{
+									Deployment: &v1alpha2.DeploymentSpec{
+										Replicas: helpers.GetPointer(int32(3)),
+									},
+								},
+							},
+						},
 						{Name: "gateway3"}: {},
 					},
 					IgnoredGatewayClasses: map[types.NamespacedName]*gatewayv1.GatewayClass{
@@ -335,9 +387,11 @@ var _ = Describe("Collector", Ordered, func() {
 						}: {},
 					},
 					ReferencedNginxProxies: map[types.NamespacedName]*graph.NginxProxy{
-						{Namespace: "test", Name: "NginxProxy-1"}: {},
-						{Namespace: "test", Name: "NginxProxy-2"}: {},
-					}, SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
+						{Namespace: "test", Name: "NginxProxy-1"}: &gcNP,
+						{Namespace: "test", Name: "NginxProxy-2"}: {Valid: true},
+						{Namespace: "test", Name: "NginxProxy-3"}: {Valid: true},
+					},
+					SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
 						{Namespace: "test", Name: "sf-1"}: {
 							Snippets: map[ngfAPI.NginxContext]string{
 								ngfAPI.NginxContextMain:               "worker_priority 0;",
@@ -432,9 +486,10 @@ var _ = Describe("Collector", Ordered, func() {
 					GatewayAttachedClientSettingsPolicyCount: 1,
 					RouteAttachedClientSettingsPolicyCount:   2,
 					ObservabilityPolicyCount:                 1,
-					NginxProxyCount:                          2,
+					NginxProxyCount:                          3,
 					SnippetsFilterCount:                      3,
 					UpstreamSettingsPolicyCount:              1,
+					GatewayAttachedNpCount:                   2,
 				}
 				expData.ClusterVersion = "1.29.2"
 				expData.ClusterPlatform = "kind"
@@ -462,6 +517,9 @@ var _ = Describe("Collector", Ordered, func() {
 					1,
 				}
 
+				expData.NginxPodCount = int64(4)
+				expData.ControlPlanePodCount = int64(2)
+
 				data, err := dataCollector.Collect(ctx)
 				Expect(err).ToNot(HaveOccurred())
 
@@ -527,7 +585,7 @@ var _ = Describe("Collector", Ordered, func() {
 						},
 					}
 
-					k8sClientReader.ListCalls(createListCallsFunc(nodes))
+					k8sClientReader.ListCalls(createListCallsFunc(nodes, podList))
 					expData.ClusterVersion = "unknown"
 					expData.ClusterPlatform = "k3s"
 
@@ -543,7 +601,7 @@ var _ = Describe("Collector", Ordered, func() {
 	Describe("node count collector", func() {
 		When("collecting node count data", func() {
 			It("collects correct data for one node", func(ctx SpecContext) {
-				k8sClientReader.ListCalls(createListCallsFunc(nodeList))
+				k8sClientReader.ListCalls(createListCallsFunc(nodeList, podList))
 
 				expData.ClusterNodeCount = 1
 
@@ -593,7 +651,7 @@ var _ = Describe("Collector", Ordered, func() {
 			svc := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "svc1"}}
 
 			graph1 = &graph.Graph{
-				GatewayClass: &graph.GatewayClass{},
+				GatewayClass: &graph.GatewayClass{NginxProxy: &graph.NginxProxy{Valid: true}},
 				Gateways: map[types.NamespacedName]*graph.Gateway{
 					{Name: "gateway1"}: {},
 				},
@@ -634,12 +692,14 @@ var _ = Describe("Collector", Ordered, func() {
 					}: {},
 				},
 				ReferencedNginxProxies: map[types.NamespacedName]*graph.NginxProxy{
-					{Namespace: "test", Name: "NginxProxy-1"}: {},
-					{Namespace: "test", Name: "NginxProxy-2"}: {},
+					{Namespace: "test", Name: "NginxProxy-1"}: {Valid: true},
 				},
 				SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
 					{Namespace: "test", Name: "sf-1"}: {},
 				},
+				BackendTLSPolicies: map[types.NamespacedName]*graph.BackendTLSPolicy{
+					{Namespace: "test", Name: "BackendTLSPolicy-1"}: {},
+				},
 			}
 
 			config1 = []*dataplane.Configuration{
@@ -716,9 +776,11 @@ var _ = Describe("Collector", Ordered, func() {
 					GatewayAttachedClientSettingsPolicyCount: 1,
 					RouteAttachedClientSettingsPolicyCount:   1,
 					ObservabilityPolicyCount:                 1,
-					NginxProxyCount:                          2,
+					NginxProxyCount:                          1,
 					SnippetsFilterCount:                      1,
 					UpstreamSettingsPolicyCount:              1,
+					GatewayAttachedNpCount:                   1,
+					BackendTLSPolicyCount:                    1,
 				}
 
 				data, err := dataCollector.Collect(ctx)
@@ -834,20 +896,6 @@ var _ = Describe("Collector", Ordered, func() {
 					Expect(err).To(MatchError(expectedErr))
 				})
 
-				It("should error if the replica set's replicas is nil", func(ctx SpecContext) {
-					expectedErr := errors.New("replica set replicas was nil")
-					k8sClientReader.GetCalls(mergeGetCallsWithBase(createGetCallsFunc(
-						&appsv1.ReplicaSet{
-							Spec: appsv1.ReplicaSetSpec{
-								Replicas: nil,
-							},
-						},
-					)))
-
-					_, err := dataCollector.Collect(ctx)
-					Expect(err).To(MatchError(expectedErr))
-				})
-
 				It("should error if the kubernetes client errored when getting the ReplicaSet", func(ctx SpecContext) {
 					expectedErr := errors.New("there was an error getting the ReplicaSet")
 					k8sClientReader.GetCalls(mergeGetCallsWithBase(

internal/mode/static/telemetry/data.avdl

@@ -102,8 +102,14 @@ attached at the Gateway level. */
 		/** UpstreamSettingsPolicyCount is the number of UpstreamSettingsPolicies. */
 		long? UpstreamSettingsPolicyCount = null;
 
-		/** NGFReplicaCount is the number of replicas of the NGF Pod. */
-		long? NGFReplicaCount = null;
+		/** GatewayAttachedNpCount is the total number of NginxProxy resources that are attached to a Gateway. */
+		long? GatewayAttachedNpCount = null;
+		
+		/** NginxPodCount is the total number of Nginx data plane Pods. */
+		long? NginxPodCount = null;
+		
+		/** ControlPlanePodCount is the total number of NGF control plane Pods. */
+		long? ControlPlanePodCount = null;
 
 	}
 }