From c5fb1d8a157d79195634e409f0c6e872aa92d213 Mon Sep 17 00:00:00 2001 From: Jayapriya Pai Date: Thu, 5 Jun 2025 13:11:26 +0530 Subject: [PATCH 1/4] CHANGE: Migrate service discovery to EndpointSlices Endpoints are deprecated in k8s 1.33, to avoid warnings and stay upto-date with upstream k8s we need to move to EndpointSlices. This commit sets the field `serviceDiscoveryRole` to `EndpointSlices` and updates the RBAC permissions on endpointslice resoures. Related-to prometheus-operator/kube-prometheus#2646 Signed-off-by: Jayapriya Pai Signed-off-by: Simon Pasquier --- .../components/prometheus.libsonnet | 20 +++++++++++++- jsonnet/kube-prometheus/main.libsonnet | 1 + manifests/prometheus-prometheus.yaml | 1 + .../prometheus-roleSpecificNamespaces.yaml | 27 ++++++++++++++++--- 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/jsonnet/kube-prometheus/components/prometheus.libsonnet b/jsonnet/kube-prometheus/components/prometheus.libsonnet index 05f86711a5..d70eb4045d 100644 --- a/jsonnet/kube-prometheus/components/prometheus.libsonnet +++ b/jsonnet/kube-prometheus/components/prometheus.libsonnet @@ -29,6 +29,7 @@ local defaults = { for labelName in std.objectFields(defaults.commonLabels) if !std.setMember(labelName, ['app.kubernetes.io/version']) }, + serviceDiscoveryRole:: 'EndpointSlice', mixin:: { ruleLabels: {}, _config: { @@ -52,6 +53,21 @@ local defaults = { reloaderPort:: 8080, }; +local endpointSliceRule = { + apiGroups: ['discovery.k8s.io'], + resources: ['endpointslices'], + verbs: ['get', 'list', 'watch'], +}; +local endpointsRule = { + apiGroups: [''], + resources: ['endpoints'], + verbs: ['get', 'list', 'watch'], +}; + +local validateServiceDiscoveryRole(role) = + if role == 'EndpointSlice' then endpointSliceRule + else if role == 'Endpoints' then endpointsRule + else error 'Invalid serviceDiscoveryRole'; function(params) { local p = self, @@ -286,9 +302,10 @@ function(params) { namespace: namespace, }, rules: [ + validateServiceDiscoveryRole(p._config.serviceDiscoveryRole), { apiGroups: [''], - resources: ['services', 'endpoints', 'pods'], + resources: ['services', 'pods'], verbs: ['get', 'list', 'watch'], }, { @@ -334,6 +351,7 @@ function(params) { podMetadata: { labels: p.prometheus.metadata.labels, }, + serviceDiscoveryRole: p._config.serviceDiscoveryRole, externalLabels: p._config.externalLabels, enableFeatures: p._config.enableFeatures, serviceAccountName: p.serviceAccount.metadata.name, diff --git a/jsonnet/kube-prometheus/main.libsonnet b/jsonnet/kube-prometheus/main.libsonnet index 240e0f6144..bb54ad00c0 100644 --- a/jsonnet/kube-prometheus/main.libsonnet +++ b/jsonnet/kube-prometheus/main.libsonnet @@ -104,6 +104,7 @@ local utils = import './lib/utils.libsonnet'; apiVersion: 'v2', }], }, + serviceDiscoveryRole: 'EndpointSlice', mixin+: { ruleLabels: $.values.common.ruleLabels }, }, prometheusAdapter: { diff --git a/manifests/prometheus-prometheus.yaml b/manifests/prometheus-prometheus.yaml index 5f3ecc024a..05fff4465a 100644 --- a/manifests/prometheus-prometheus.yaml +++ b/manifests/prometheus-prometheus.yaml @@ -45,6 +45,7 @@ spec: runAsNonRoot: true runAsUser: 1000 serviceAccountName: prometheus-k8s + serviceDiscoveryRole: EndpointSlice serviceMonitorNamespaceSelector: {} serviceMonitorSelector: {} version: 3.6.0 diff --git a/manifests/prometheus-roleSpecificNamespaces.yaml b/manifests/prometheus-roleSpecificNamespaces.yaml index 2fb927aaf4..cdb766fc6e 100644 --- a/manifests/prometheus-roleSpecificNamespaces.yaml +++ b/manifests/prometheus-roleSpecificNamespaces.yaml @@ -12,11 +12,18 @@ items: name: prometheus-k8s namespace: default rules: + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch - apiGroups: - "" resources: - services - - endpoints - pods verbs: - get @@ -50,11 +57,18 @@ items: name: prometheus-k8s namespace: kube-system rules: + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch - apiGroups: - "" resources: - services - - endpoints - pods verbs: - get @@ -88,11 +102,18 @@ items: name: prometheus-k8s namespace: monitoring rules: + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch - apiGroups: - "" resources: - services - - endpoints - pods verbs: - get From 2407c563b122b078206c47c8ed191ff9240befe4 Mon Sep 17 00:00:00 2001 From: Jayapriya Pai Date: Tue, 17 Jun 2025 11:56:01 +0530 Subject: [PATCH 2/4] CHANGE: set `--kubelet-endpointslice` in operator Signed-off-by: Jayapriya Pai --- .../components/prometheus-operator.libsonnet | 2 ++ manifests/prometheusOperator-clusterRole.yaml | 10 ++++++++++ manifests/prometheusOperator-deployment.yaml | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet b/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet index 104d76a8ba..c7c142885f 100644 --- a/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet +++ b/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet @@ -9,6 +9,8 @@ local defaults = { namespace:: error 'must provide namespace', version:: error 'must provide version', image:: error 'must provide image', + kubeletEndpointsEnabled:: true, + kubeletEndpointSliceEnabled:: true, kubeRbacProxyImage:: error 'must provide kubeRbacProxyImage', configReloaderImage:: error 'must provide config reloader image', resources:: { diff --git a/manifests/prometheusOperator-clusterRole.yaml b/manifests/prometheusOperator-clusterRole.yaml index 6ca381ef60..4729cbc7b6 100644 --- a/manifests/prometheusOperator-clusterRole.yaml +++ b/manifests/prometheusOperator-clusterRole.yaml @@ -109,6 +109,16 @@ rules: - create - update - delete +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - create + - list + - update + - delete - apiGroups: - authentication.k8s.io resources: diff --git a/manifests/prometheusOperator-deployment.yaml b/manifests/prometheusOperator-deployment.yaml index 0d45924f8d..ffad891b7e 100644 --- a/manifests/prometheusOperator-deployment.yaml +++ b/manifests/prometheusOperator-deployment.yaml @@ -31,7 +31,7 @@ spec: - --kubelet-service=kube-system/kubelet - --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.85.0 - --kubelet-endpoints=true - - --kubelet-endpointslice=false + - --kubelet-endpointslice=true env: - name: GOGC value: "30" From 27c3f4d9e31d56928d268a4acd4eefb002ebb20c Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 2 Oct 2025 16:18:05 +0200 Subject: [PATCH 3/4] refactor jsonnet Signed-off-by: Simon Pasquier --- .../components/prometheus-operator.libsonnet | 4 +++ .../components/prometheus.libsonnet | 31 +++++++++---------- jsonnet/kube-prometheus/main.libsonnet | 1 - 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet b/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet index c7c142885f..9f5c713008 100644 --- a/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet +++ b/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet @@ -9,6 +9,10 @@ local defaults = { namespace:: error 'must provide namespace', version:: error 'must provide version', image:: error 'must provide image', + // Enable both endpoints and endpointslice for the kubelet controller to + // ensure that the `endpointslice.kubernetes.io/skip-mirror=true` label is + // set on the endpoints object. In the next release we can turn off endpoints + // management. kubeletEndpointsEnabled:: true, kubeletEndpointSliceEnabled:: true, kubeRbacProxyImage:: error 'must provide kubeRbacProxyImage', diff --git a/jsonnet/kube-prometheus/components/prometheus.libsonnet b/jsonnet/kube-prometheus/components/prometheus.libsonnet index d70eb4045d..cedd46416b 100644 --- a/jsonnet/kube-prometheus/components/prometheus.libsonnet +++ b/jsonnet/kube-prometheus/components/prometheus.libsonnet @@ -53,22 +53,6 @@ local defaults = { reloaderPort:: 8080, }; -local endpointSliceRule = { - apiGroups: ['discovery.k8s.io'], - resources: ['endpointslices'], - verbs: ['get', 'list', 'watch'], -}; -local endpointsRule = { - apiGroups: [''], - resources: ['endpoints'], - verbs: ['get', 'list', 'watch'], -}; - -local validateServiceDiscoveryRole(role) = - if role == 'EndpointSlice' then endpointSliceRule - else if role == 'Endpoints' then endpointsRule - else error 'Invalid serviceDiscoveryRole'; - function(params) { local p = self, _config:: defaults + params, @@ -302,7 +286,20 @@ function(params) { namespace: namespace, }, rules: [ - validateServiceDiscoveryRole(p._config.serviceDiscoveryRole), + if p._config.serviceDiscoveryRole == 'EndpointSlice' then { + apiGroups: ['discovery.k8s.io'], + resources: ['endpointslices'], + verbs: ['get', 'list', 'watch'], + } + else if p._config.serviceDiscoveryRole == 'Endpoints' then + { + apiGroups: [''], + resources: ['endpoints'], + verbs: ['get', 'list', 'watch'], + } + else + error 'Invalid serviceDiscoveryRole: ' + p._config.serviceDiscoveryRole, + ] + [ { apiGroups: [''], resources: ['services', 'pods'], diff --git a/jsonnet/kube-prometheus/main.libsonnet b/jsonnet/kube-prometheus/main.libsonnet index bb54ad00c0..240e0f6144 100644 --- a/jsonnet/kube-prometheus/main.libsonnet +++ b/jsonnet/kube-prometheus/main.libsonnet @@ -104,7 +104,6 @@ local utils = import './lib/utils.libsonnet'; apiVersion: 'v2', }], }, - serviceDiscoveryRole: 'EndpointSlice', mixin+: { ruleLabels: $.values.common.ruleLabels }, }, prometheusAdapter: { From 3c52f282d8c4c47f5bfeef03327a6bfddab77432 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 2 Oct 2025 17:08:55 +0200 Subject: [PATCH 4/4] docs: add endpoints migration Signed-off-by: Simon Pasquier --- docs/endpoints-migration.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docs/endpoints-migration.md diff --git a/docs/endpoints-migration.md b/docs/endpoints-migration.md new file mode 100644 index 0000000000..fb71b185df --- /dev/null +++ b/docs/endpoints-migration.md @@ -0,0 +1,16 @@ +# Migration from Endpoints to EndpointSlice + +`kube-prometheus` 0.17+ automatically configures Prometheus to use EndpointSlice instead of Endpoints for Kubernetes service discovery (Endpoints have been deprecated in Kubernetes 1.33). + +While the migration should be seamless for "regular" pods, it requires a few manual steps for components running as host services (e.g. node_exporter and kubelet): +1. The node_exporter and kubelet ServiceMonitors rely on the Prometheus operator's kubelet controller which manages the `kube-system/kubelet` Service. +2. With `kube-prometheus` 0.17, the Prometheus operator starts with both `--kubelet-endpoints=true` and `--kubelet-endpointslice=true` to ensure that a) the operator synchronizes the EndpointSlice object(s) backing the `kube-system/kubelet` Service and b) Kubernetes stops mirroring the `kube-system/kubelet` Endpoints object to EndpointSlice object(s) (otherwise the operator and kube-controller-manager would fight for the same resources). +3. After verifying that all targets are correctly discovered, it is ok to modify the operator's deployment and use `--kubelet-endpoints=false` instead. This will become the default in a future version of `kube-prometheus`. +4. The `kube-system/kubelet` Endpoints object should be removed manually. + +To verify the status of the Endpoints and EndpointSlice objects, run: + +```shell +kubectl get -n kube-system endpoints kubelet +kubectl get -n kube-system endpointslice -l endpointslice.kubernetes.io/managed-by=prometheus-operator +```