feat: expose and secure controller-runtime metrics

.gitignore

@@ -362,3 +362,4 @@ admin/Cargo.lock
 *.csr
 *.srl
 *.ext
+*.test

CHANGELOG.md

@@ -45,6 +45,12 @@ This changelog keeps track of work items that have been completed and are ready
 
 - **General**: TODO ([#TODO](https://github.com/kedacore/http-add-on/issues/TODO))
 
+## v0.12.0
+
+### Improvements
+
+- **General**: Get rid of kube-rbac-proxy ([#1123](https://github.com/kedacore/http-add-on/pull/1369))
+
 ## v0.11.1
 
 ### Improvements

config/operator/deployment.yaml

@@ -38,7 +38,7 @@ spec:
           value: ""
         ports:
         - name: metrics
-          containerPort: 8080
+          containerPort: 8443
         - name: probes
           containerPort: 8081
         livenessProbe:

config/operator/kustomization.yaml

@@ -2,6 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - deployment.yaml
+- metrics_service.yaml
+- metrics_reader_role.yaml
 - role.yaml
 - role_binding.yaml
 - service_account.yaml

config/operator/metrics_reader_role.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: operator-metrics-reader
+rules:
+- nonResourceURLs:
+  - "/metrics"
+  verbs:
+  - get

config/operator/metrics_service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: operator-metrics
+spec:
+  type: ClusterIP
+  ports:
+  - name: metrics
+    protocol: TCP
+    port: 8443
+    targetPort: 8443

config/operator/role.yaml

@@ -42,6 +42,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

config/operator/role_binding.yaml

@@ -2,23 +2,39 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: operator
+  name: keda-http-operator-auth-delegator
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: operator
+  name: system:auth-delegator
 subjects:
 - kind: ServiceAccount
+  name: keda-add-ons-http-operator
+  namespace: keda
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: keda-http-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
   name: operator
+subjects:
+- kind: ServiceAccount
+  name: keda-add-ons-http-operator
+  namespace: keda
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: operator
+  namespace: keda
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: operator
 subjects:
 - kind: ServiceAccount
-  name: operator
+  name: keda-add-ons-http-operator
+  namespace: keda

go.mod

@@ -60,6 +60,16 @@ require (
 	sigs.k8s.io/kustomize/kustomize/v5 v5.7.1
 )
 
+require (
+	cel.dev/expr v0.24.0 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
+	github.com/google/cel-go v0.26.0 // indirect
+	github.com/stoewer/go-strcase v1.3.1 // indirect
+	k8s.io/apiserver v0.33.5 // indirect
+	k8s.io/component-base v0.33.5 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
+)
+
 require (
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect

go.sum

@@ -1,5 +1,9 @@
+cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
+cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
+github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -64,6 +68,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
 github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
+github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g=
+github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8=
 github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
 github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -172,6 +178,8 @@ github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wx
 github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
 github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs=
+github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -328,10 +336,14 @@ k8s.io/apiextensions-apiserver v0.33.5 h1:93NZh6rmrcamX/tfv/dZrTsMiQX69ufANmDcKP
 k8s.io/apiextensions-apiserver v0.33.5/go.mod h1:JIbyQnNlu6nQa7b1vgFi51pmlXOk8mdn0WJwUJnz/7U=
 k8s.io/apimachinery v0.33.5 h1:NiT64hln4TQXeYR18/ES39OrNsjGz8NguxsBgp+6QIo=
 k8s.io/apimachinery v0.33.5/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
+k8s.io/apiserver v0.33.5 h1:X1Gy33r4YkRLRqTjGjofk7X1/EjSLEVSJ/A+1qjoj60=
+k8s.io/apiserver v0.33.5/go.mod h1:Q+b5Btbc8x0PqOCeh/xBTesKk+cXQRN+PF2wdrTKDeg=
 k8s.io/client-go v0.33.5 h1:I8BdmQGxInpkMEnJvV6iG7dqzP3JRlpZZlib3OMFc3o=
 k8s.io/client-go v0.33.5/go.mod h1:W8PQP4MxbM4ypgagVE65mUUqK1/ByQkSALF9tzuQ6u0=
 k8s.io/code-generator v0.33.5 h1:KwkOvhwAaorjSwF2MQhhdhL3i8bBmAal/TWhX67kdHw=
 k8s.io/code-generator v0.33.5/go.mod h1:Ra+sdZquRakeTGcEnQAPw6BmlZ92IvxwQQTX/XOvOIE=
+k8s.io/component-base v0.33.5 h1:4D3kxjEx1pJRy3WHAZsmX3+LCpmd4ftE+2J4v6naTnQ=
+k8s.io/component-base v0.33.5/go.mod h1:Zma1YjBVuuGxIbspj1vGR3/5blzo2ARf1v0QTtog1to=
 k8s.io/gengo/v2 v2.0.0-20250820003526-c297c0c1eb9d h1:qUrYOinhdAUL0xxhA4gPqogPBaS9nIq2l2kTb6pmeB0=
 k8s.io/gengo/v2 v2.0.0-20250820003526-c297c0c1eb9d/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
@@ -342,6 +354,8 @@ k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzk
 k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 knative.dev/pkg v0.0.0-20250602175424-3c3a920206ea h1:ukJPq9MzFTEH/Sei5MSVnSE8+7NSCKixCDZPd6p4ohw=
 knative.dev/pkg v0.0.0-20250602175424-3c3a920206ea/go.mod h1:tFayQbi6t4+5HXuEGLOGvILW228Q7uaJp/FYEgbjJ3A=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
 sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8=
 sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM=
 sigs.k8s.io/controller-tools v0.17.3 h1:lwFPLicpBKLgIepah+c8ikRBubFW5kOQyT88r3EwfNw=

operator/main.go

@@ -29,6 +29,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	httpv1alpha1 "github.com/kedacore/http-add-on/operator/apis/http/v1alpha1"
@@ -58,7 +59,7 @@ func main() {
 	var enableLeaderElection bool
 	var probeAddr string
 	var profilingAddr string
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8443", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -96,7 +97,9 @@ func main() {
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme: scheme,
 		Metrics: server.Options{
-			BindAddress: metricsAddr,
+			BindAddress:    metricsAddr,
+			SecureServing:  true,
+			FilterProvider: filters.WithAuthenticationAndAuthorization,
 		},
 		PprofBindAddress:              profilingAddr,
 		HealthProbeBindAddress:        probeAddr,

tests/checks/operator_metrics/operator_metrics_test.go

@@ -0,0 +1,140 @@
+//go:build e2e
+// +build e2e
+
+package operator_metrics_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	. "github.com/kedacore/http-add-on/tests/helper"
+)
+
+const (
+	testName = "operator-metrics-test"
+)
+
+var (
+	testNamespace          = fmt.Sprintf("%s-ns", testName)
+	clientName             = fmt.Sprintf("%s-client", testName)
+	kedaOperatorMetricsURL = "https://keda-add-ons-http-operator-metrics.keda:8443/metrics"
+	operatorPodSelector    = "app.kubernetes.io/instance=operator"
+)
+
+type templateData struct {
+	TestNamespace string
+	ClientName    string
+}
+
+const (
+	clientTemplate = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{.ClientName}}
+  namespace: {{.TestNamespace}}
+spec:
+  serviceAccountName: {{.ClientName}}
+  containers:
+  - name: {{.ClientName}}
+    image: curlimages/curl
+    command:
+      - sh
+      - -c
+      - "exec tail -f /dev/null"`
+
+	serviceAccountTemplate = `
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{.ClientName}}
+  namespace: {{.TestNamespace}}`
+
+	clusterRoleBindingTemplate = `
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{.ClientName}}-metrics-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: keda-add-ons-http-operator-metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: operator-metrics-test-client
+  namespace: {{.TestNamespace}}`
+)
+
+func TestOperatorMetrics(t *testing.T) {
+	// setup
+	t.Log("--- setting up ---")
+	// Create kubernetes resources
+	kc := GetKubernetesClient(t)
+	data, templates := getTemplateData()
+	CreateKubernetesResources(t, kc, testNamespace, data, templates)
+
+	// Wait for client pod to be ready
+	assert.True(t, WaitForAllPodRunningInNamespace(t, kc, testNamespace, 6, 10),
+		"client pod should be running")
+
+	t.Log("--- testing operator metrics endpoint ---")
+
+	// Test 1: HTTPS endpoint should be accessible (will fail cert validation but should return metrics)
+	t.Log("Test 1: Verify HTTPS endpoint is available")
+	testHTTPSEndpoint(t)
+
+	// Test 2: Verify metrics are returned
+	t.Log("Test 2: Verify metrics content")
+	testMetricsContent(t)
+
+	// cleanup
+	DeleteKubernetesResources(t, testNamespace, data, templates)
+}
+
+func testHTTPSEndpoint(t *testing.T) {
+	// Use curl with -k to skip certificate validation (self-signed cert)
+	cmd := fmt.Sprintf("curl -k --max-time 10 %s", kedaOperatorMetricsURL)
+	out, errOut, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd)
+
+	// We expect this to succeed with a self-signed certificate
+	if err != nil {
+		t.Logf("HTTPS endpoint test - Output: %s, Error output: %s, Error: %v", out, errOut, err)
+	}
+
+	// The endpoint should return something (even if authentication fails, it should respond)
+	assert.True(t, err == nil || strings.Contains(errOut, "Forbidden") || strings.Contains(out, "Forbidden"),
+		"HTTPS endpoint should respond (either with metrics or authentication error)")
+}
+
+func testMetricsContent(t *testing.T) {
+	// Access metrics from the client pod using the service endpoint
+	// The client pod uses a ServiceAccount with the operator-metrics-reader ClusterRole
+	// This allows it to access the metrics endpoint with proper RBAC permissions
+
+	// Get the ServiceAccount token to authenticate
+	cmd := fmt.Sprintf("curl -k -H \"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" --max-time 10 %s", kedaOperatorMetricsURL)
+	out, errOut, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd)
+
+	if err != nil {
+		t.Logf("Metrics content test - Output: %s, Error output: %s, Error: %v", out, errOut, err)
+	}
+
+	// Verify that metrics are returned in Prometheus format
+	assert.NoError(t, err, "should be able to access metrics from client pod with RBAC permissions. Output: %s, Error: %s", out, errOut)
+	assert.True(t, strings.Contains(out, "# HELP") || strings.Contains(out, "# TYPE"),
+		"metrics should contain Prometheus format. Output: %s", out)
+}
+
+func getTemplateData() (templateData, []Template) {
+	return templateData{
+			TestNamespace: testNamespace,
+			ClientName:    clientName,
+		}, []Template{
+			{Name: "serviceAccountTemplate", Config: serviceAccountTemplate},
+			{Name: "clusterRoleBindingTemplate", Config: clusterRoleBindingTemplate},
+			{Name: "clientTemplate", Config: clientTemplate},
+		}
+}

tests/utils/setup_test.go

@@ -66,7 +66,13 @@ kind: Deployment
 metadata:
   name: opentelemetry-collector
 spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: opentelemetry-collector
   template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: opentelemetry-collector
     spec:
       containers:
       - name: opentelemetry-collector