CNTRLPLANE-3584: Enable resource metrics for kube-scheduler endpoints

control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/servicemonitor.yaml

@@ -24,6 +24,22 @@ spec:
         key: tls.key
         name: metrics-client
       serverName: kube-scheduler
+  - scheme: https
+    path: /metrics/resources
+    targetPort: client
+    tlsConfig:
+      ca:
+        configMap:
+          key: ca.crt
+          name: root-ca
+      cert:
+        secret:
+          key: tls.crt
+          name: metrics-client
+      keySecret:
+        key: tls.key
+        name: metrics-client
+      serverName: kube-scheduler
   selector:
     matchLabels:
       app: kube-scheduler

control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor.go

@@ -12,8 +12,12 @@ func adaptServiceMonitor(cpContext component.WorkloadContext, sm *prometheusoper
 	sm.Spec.NamespaceSelector = prometheusoperatorv1.NamespaceSelector{
 		MatchNames: []string{sm.Namespace},
 	}
+
 	sm.Spec.Endpoints[0].MetricRelabelConfigs = metrics.SchedulerRelabelConfigs(cpContext.MetricsSet)
 	util.ApplyClusterIDLabel(&sm.Spec.Endpoints[0], cpContext.HCP.Spec.ClusterID)
 
+	sm.Spec.Endpoints[1].MetricRelabelConfigs = metrics.SchedulerResourceRelabelConfigs(cpContext.MetricsSet)
+	util.ApplyClusterIDLabel(&sm.Spec.Endpoints[1], cpContext.HCP.Spec.ClusterID)
+
 	return nil
 }

control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor_test.go

@@ -34,37 +34,55 @@ func TestAdaptServiceMonitor(t *testing.T) {
 			},
 		},
 		{
-			name:       "When service monitor is adapted it should apply cluster ID label",
+			name:       "When service monitor is adapted it should apply cluster ID label to both endpoints",
 			metricsSet: metrics.MetricsSetAll,
 			clusterID:  "cluster-abc-123",
 			validate: func(t *testing.T, sm *prometheusoperatorv1.ServiceMonitor, err error) {
 				g := NewWithT(t)
 				g.Expect(err).ToNot(HaveOccurred())
-				g.Expect(sm.Spec.Endpoints).To(HaveLen(1))
-
-				relabelConfigs := sm.Spec.Endpoints[0].RelabelConfigs
-				foundClusterIDLabel := false
-				for _, config := range relabelConfigs {
-					if config.TargetLabel == "_id" && config.Replacement != nil && *config.Replacement == "cluster-abc-123" {
-						foundClusterIDLabel = true
-						break
+				g.Expect(sm.Spec.Endpoints).To(HaveLen(2))
+
+				for i, ep := range sm.Spec.Endpoints {
+					foundClusterIDLabel := false
+					for _, config := range ep.RelabelConfigs {
+						if config.TargetLabel == "_id" && config.Replacement != nil && *config.Replacement == "cluster-abc-123" {
+							foundClusterIDLabel = true
+							break
+						}
 					}
+					g.Expect(foundClusterIDLabel).To(BeTrue(), "cluster ID label should be applied to endpoint %d", i)
 				}
-				g.Expect(foundClusterIDLabel).To(BeTrue(), "cluster ID label should be applied")
 			},
 		},
 		{
-			name:       "When metrics set is Telemetry it should drop all metrics",
+			name:       "When metrics set is Telemetry it should drop all metrics on both endpoints",
 			metricsSet: metrics.MetricsSetTelemetry,
 			clusterID:  "test-cluster",
 			validate: func(t *testing.T, sm *prometheusoperatorv1.ServiceMonitor, err error) {
 				g := NewWithT(t)
 				g.Expect(err).ToNot(HaveOccurred())
-				g.Expect(sm.Spec.Endpoints).To(HaveLen(1))
+				g.Expect(sm.Spec.Endpoints).To(HaveLen(2))
 
 				g.Expect(sm.Spec.Endpoints[0].MetricRelabelConfigs).To(HaveLen(2))
 				g.Expect(sm.Spec.Endpoints[0].MetricRelabelConfigs[0].Action).To(Equal("drop"))
 				g.Expect(sm.Spec.Endpoints[0].MetricRelabelConfigs[0].Regex).To(Equal(".*"))
+
+				g.Expect(sm.Spec.Endpoints[1].MetricRelabelConfigs).To(HaveLen(2))
+				g.Expect(sm.Spec.Endpoints[1].MetricRelabelConfigs[0].Action).To(Equal("drop"))
+				g.Expect(sm.Spec.Endpoints[1].MetricRelabelConfigs[0].Regex).To(Equal(".*"))
+			},
+		},
+		{
+			name:       "When metrics set is All it should not add metric relabel configs on the resources endpoint",
+			metricsSet: metrics.MetricsSetAll,
+			clusterID:  "test-cluster",
+			validate: func(t *testing.T, sm *prometheusoperatorv1.ServiceMonitor, err error) {
+				g := NewWithT(t)
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(sm.Spec.Endpoints).To(HaveLen(2))
+
+				g.Expect(sm.Spec.Endpoints[1].MetricRelabelConfigs).To(HaveLen(1))
+				g.Expect(sm.Spec.Endpoints[1].MetricRelabelConfigs[0].TargetLabel).To(Equal("_id"))
 			},
 		},
 	}
@@ -99,6 +117,10 @@ func TestAdaptServiceMonitor(t *testing.T) {
 						{
 							Port: "metrics",
 						},
+						{
+							Port: "metrics",
+							Path: "/metrics/resources",
+						},
 					},
 				},
 			}

control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests/rbac.go

@@ -137,6 +137,22 @@ func MetricsClientClusterRoleBinding() *rbacv1.ClusterRoleBinding {
 	}
 }
 
+func MetricsResourcesClusterRole() *rbacv1.ClusterRole {
+	return &rbacv1.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "hypershift-metrics-resources-reader",
+		},
+	}
+}
+
+func MetricsResourcesClusterRoleBinding() *rbacv1.ClusterRoleBinding {
+	return &rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "hypershift-metrics-resources-reader",
+		},
+	}
+}
+
 func AuthenticatedReaderForAuthenticatedUserRolebinding() *rbacv1.RoleBinding {
 	return &rbacv1.RoleBinding{
 		ObjectMeta: metav1.ObjectMeta{

control-plane-operator/hostedclusterconfigoperator/controllers/resources/rbac/reconcile.go

@@ -280,6 +280,32 @@ func ReconcileGenericMetricsClusterRoleBinding(cn string) func(*rbacv1.ClusterRo
 	}
 }
 
+func ReconcileMetricsResourcesClusterRole(r *rbacv1.ClusterRole) error {
+	r.Rules = []rbacv1.PolicyRule{
+		{
+			NonResourceURLs: []string{"/metrics/resources"},
+			Verbs:           []string{"get"},
+		},
+	}
+	return nil
+}
+
+func ReconcileMetricsResourcesClusterRoleBinding(r *rbacv1.ClusterRoleBinding) error {
+	r.RoleRef = rbacv1.RoleRef{
+		APIGroup: rbacv1.SchemeGroupVersion.Group,
+		Kind:     "ClusterRole",
+		Name:     "hypershift-metrics-resources-reader",
+	}
+	r.Subjects = []rbacv1.Subject{
+		{
+			APIGroup: rbacv1.SchemeGroupVersion.Group,
+			Kind:     "User",
+			Name:     "system:serviceaccount:hypershift:prometheus",
+		},
+	}
+	return nil
+}
+
 func ReconcileAuthenticatedReaderForAuthenticatedUserRolebinding(r *rbacv1.RoleBinding) error {
 	r.RoleRef = rbacv1.RoleRef{
 		APIGroup: rbacv1.SchemeGroupVersion.Group,

control-plane-operator/hostedclusterconfigoperator/controllers/resources/rbac/reconcile_test.go

@@ -0,0 +1,51 @@
+package rbac
+
+import (
+	"testing"
+
+	rbacv1 "k8s.io/api/rbac/v1"
+)
+
+func TestReconcileMetricsResourcesClusterRole(t *testing.T) {
+	t.Run("When reconciling it should set the correct policy rules", func(t *testing.T) {
+		role := &rbacv1.ClusterRole{}
+		if err := ReconcileMetricsResourcesClusterRole(role); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if len(role.Rules) != 1 {
+			t.Fatalf("expected 1 rule, got %d", len(role.Rules))
+		}
+		rule := role.Rules[0]
+		if len(rule.NonResourceURLs) != 1 || rule.NonResourceURLs[0] != "/metrics/resources" {
+			t.Errorf("expected NonResourceURLs [\"/metrics/resources\"], got %v", rule.NonResourceURLs)
+		}
+		if len(rule.Verbs) != 1 || rule.Verbs[0] != "get" {
+			t.Errorf("expected Verbs [\"get\"], got %v", rule.Verbs)
+		}
+	})
+}
+
+func TestReconcileMetricsResourcesClusterRoleBinding(t *testing.T) {
+	t.Run("When reconciling it should set the correct role ref and subjects", func(t *testing.T) {
+		binding := &rbacv1.ClusterRoleBinding{}
+		if err := ReconcileMetricsResourcesClusterRoleBinding(binding); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if binding.RoleRef.Kind != "ClusterRole" {
+			t.Errorf("expected RoleRef.Kind ClusterRole, got %s", binding.RoleRef.Kind)
+		}
+		if binding.RoleRef.Name != "hypershift-metrics-resources-reader" {
+			t.Errorf("expected RoleRef.Name hypershift-metrics-resources-reader, got %s", binding.RoleRef.Name)
+		}
+		if len(binding.Subjects) != 1 {
+			t.Fatalf("expected 1 subject, got %d", len(binding.Subjects))
+		}
+		subject := binding.Subjects[0]
+		if subject.Kind != "User" {
+			t.Errorf("expected subject Kind User, got %s", subject.Kind)
+		}
+		if subject.Name != "system:serviceaccount:hypershift:prometheus" {
+			t.Errorf("expected subject Name system:serviceaccount:hypershift:prometheus, got %s", subject.Name)
+		}
+	})
+}

control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go

@@ -1333,6 +1333,8 @@ func (r *reconciler) reconcileRBAC(ctx context.Context, hcp *hyperv1.HostedContr
 		manifestAndReconcile[*rbacv1.ClusterRoleBinding]{manifest: manifests.NodeBootstrapperClusterRoleBinding, reconcile: rbac.ReconcileNodeBootstrapperClusterRoleBinding},
 		manifestAndReconcile[*rbacv1.ClusterRoleBinding]{manifest: manifests.CSRRenewalClusterRoleBinding, reconcile: rbac.ReconcileCSRRenewalClusterRoleBinding},
 		manifestAndReconcile[*rbacv1.ClusterRoleBinding]{manifest: manifests.MetricsClientClusterRoleBinding, reconcile: rbac.ReconcileGenericMetricsClusterRoleBinding("system:serviceaccount:hypershift:prometheus")},
+		manifestAndReconcile[*rbacv1.ClusterRole]{manifest: manifests.MetricsResourcesClusterRole, reconcile: rbac.ReconcileMetricsResourcesClusterRole},
+		manifestAndReconcile[*rbacv1.ClusterRoleBinding]{manifest: manifests.MetricsResourcesClusterRoleBinding, reconcile: rbac.ReconcileMetricsResourcesClusterRoleBinding},
 
 		manifestAndReconcile[*rbacv1.RoleBinding]{manifest: manifests.IngressToRouteControllerRoleBinding, reconcile: rbac.ReconcileIngressToRouteControllerRoleBinding},