Migrate service discovery to EndpointSlices

docs/endpoints-migration.md

@@ -0,0 +1,16 @@
+# Migration from Endpoints to EndpointSlice
+
+`kube-prometheus` 0.17+ automatically configures Prometheus to use EndpointSlice instead of Endpoints for Kubernetes service discovery (Endpoints have been deprecated in Kubernetes 1.33).
+
+While the migration should be seamless for "regular" pods, it requires a few manual steps for components running as host services (e.g. node_exporter and kubelet):
+1. The node_exporter and kubelet ServiceMonitors rely on the Prometheus operator's kubelet controller which manages the `kube-system/kubelet` Service.
+2. With `kube-prometheus` 0.17, the Prometheus operator starts with both `--kubelet-endpoints=true` and `--kubelet-endpointslice=true` to ensure that a) the operator synchronizes the EndpointSlice object(s) backing the `kube-system/kubelet` Service and b) Kubernetes stops mirroring the `kube-system/kubelet` Endpoints object to EndpointSlice object(s) (otherwise the operator and kube-controller-manager would fight for the same resources).
+3. After verifying that all targets are correctly discovered, it is ok to modify the operator's deployment and use `--kubelet-endpoints=false` instead. This will become the default in a future version of `kube-prometheus`.
+4. The `kube-system/kubelet` Endpoints object should be removed manually.
+
+To verify the status of the Endpoints and EndpointSlice objects, run:
+
+```shell
+kubectl get -n kube-system endpoints kubelet
+kubectl get -n kube-system endpointslice -l endpointslice.kubernetes.io/managed-by=prometheus-operator
+```

jsonnet/kube-prometheus/components/prometheus-operator.libsonnet

@@ -9,6 +9,12 @@ local defaults = {
   namespace:: error 'must provide namespace',
   version:: error 'must provide version',
   image:: error 'must provide image',
+  // Enable both endpoints and endpointslice for the kubelet controller to
+  // ensure that the `endpointslice.kubernetes.io/skip-mirror=true` label is
+  // set on the endpoints object. In the next release we can turn off endpoints
+  // management.
+  kubeletEndpointsEnabled:: true,
+  kubeletEndpointSliceEnabled:: true,
   kubeRbacProxyImage:: error 'must provide kubeRbacProxyImage',
   configReloaderImage:: error 'must provide config reloader image',
   resources:: {

jsonnet/kube-prometheus/components/prometheus.libsonnet

@@ -29,6 +29,7 @@ local defaults = {
     for labelName in std.objectFields(defaults.commonLabels)
     if !std.setMember(labelName, ['app.kubernetes.io/version'])
   },
+  serviceDiscoveryRole:: 'EndpointSlice',
   mixin:: {
     ruleLabels: {},
     _config: {
@@ -52,7 +53,6 @@ local defaults = {
   reloaderPort:: 8080,
 };
 
-
 function(params) {
   local p = self,
   _config:: defaults + params,
@@ -286,9 +286,23 @@ function(params) {
         namespace: namespace,
       },
       rules: [
+        if p._config.serviceDiscoveryRole == 'EndpointSlice' then {
+          apiGroups: ['discovery.k8s.io'],
+          resources: ['endpointslices'],
+          verbs: ['get', 'list', 'watch'],
+        }
+        else if p._config.serviceDiscoveryRole == 'Endpoints' then
+          {
+            apiGroups: [''],
+            resources: ['endpoints'],
+            verbs: ['get', 'list', 'watch'],
+          }
+        else
+          error 'Invalid serviceDiscoveryRole: ' + p._config.serviceDiscoveryRole,
+      ] + [
         {
           apiGroups: [''],
-          resources: ['services', 'endpoints', 'pods'],
+          resources: ['services', 'pods'],
           verbs: ['get', 'list', 'watch'],
         },
         {
@@ -334,6 +348,7 @@ function(params) {
       podMetadata: {
         labels: p.prometheus.metadata.labels,
       },
+      serviceDiscoveryRole: p._config.serviceDiscoveryRole,
       externalLabels: p._config.externalLabels,
       enableFeatures: p._config.enableFeatures,
       serviceAccountName: p.serviceAccount.metadata.name,

manifests/prometheus-prometheus.yaml

@@ -45,6 +45,7 @@ spec:
     runAsNonRoot: true
     runAsUser: 1000
   serviceAccountName: prometheus-k8s
+  serviceDiscoveryRole: EndpointSlice
   serviceMonitorNamespaceSelector: {}
   serviceMonitorSelector: {}
   version: 3.6.0

manifests/prometheus-roleSpecificNamespaces.yaml

@@ -12,11 +12,18 @@ items:
     name: prometheus-k8s
     namespace: default
   rules:
+  - apiGroups:
+    - discovery.k8s.io
+    resources:
+    - endpointslices
+    verbs:
+    - get
+    - list
+    - watch
   - apiGroups:
     - ""
     resources:
     - services
-    - endpoints
     - pods
     verbs:
     - get
@@ -50,11 +57,18 @@ items:
     name: prometheus-k8s
     namespace: kube-system
   rules:
+  - apiGroups:
+    - discovery.k8s.io
+    resources:
+    - endpointslices
+    verbs:
+    - get
+    - list
+    - watch
   - apiGroups:
     - ""
     resources:
     - services
-    - endpoints
     - pods
     verbs:
     - get
@@ -88,11 +102,18 @@ items:
     name: prometheus-k8s
     namespace: monitoring
   rules:
+  - apiGroups:
+    - discovery.k8s.io
+    resources:
+    - endpointslices
+    verbs:
+    - get
+    - list
+    - watch
   - apiGroups:
     - ""
     resources:
     - services
-    - endpoints
     - pods
     verbs:
     - get

manifests/prometheusOperator-clusterRole.yaml

@@ -109,6 +109,16 @@ rules:
   - create
   - update
   - delete
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - create
+  - list
+  - update
+  - delete
 - apiGroups:
   - authentication.k8s.io
   resources:

manifests/prometheusOperator-deployment.yaml

@@ -31,7 +31,7 @@ spec:
         - --kubelet-service=kube-system/kubelet
         - --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.85.0
         - --kubelet-endpoints=true
-        - --kubelet-endpointslice=false
+        - --kubelet-endpointslice=true
         env:
         - name: GOGC
           value: "30"