diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go b/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go index a6c5815d9fde..232ab6a2550e 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go @@ -102,6 +102,15 @@ func (r *HostedControlPlaneReconciler) setupKASClientSigners( return err } + // kube-scheduler server cert + if _, err := reconcileSub( + manifests.KubeSchedulerServerCertSecret(hcp.Namespace), + kubeControlPlaneSigner, + pki.ReconcileKubeSchedulerServerCertSecret, + ); err != nil { + return err + } + // KCM client cert if _, err := reconcileSub( manifests.KubeControllerManagerClientCertSecret(hcp.Namespace), diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go index 20ebb285975f..f54e0dd1d11c 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go @@ -225,6 +225,10 @@ func KubeSchedulerClientCertSecret(ns string) *corev1.Secret { return secretFor(ns, "kube-scheduler-client") } +func KubeSchedulerServerCertSecret(ns string) *corev1.Secret { + return secretFor(ns, "kube-scheduler-server") +} + func KubeControllerManagerClientCertSecret(ns string) *corev1.Secret { return secretFor(ns, "kube-controller-manager") } diff --git a/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go b/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go index 60cb0b23109b..3e8cd6e0da37 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go +++ b/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go @@ -57,6 +57,16 @@ func ReconcileKubeSchedulerClientCertSecret(secret, ca *corev1.Secret, ownerRef return reconcileSignedCert(secret, ca, ownerRef, "system:kube-scheduler", []string{"kubernetes"}, X509UsageClientAuth) } +func ReconcileKubeSchedulerServerCertSecret(secret, ca *corev1.Secret, ownerRef config.OwnerRef) error { + svcName := "kube-scheduler" + dnsNames := []string{ + svcName, + fmt.Sprintf("%s.%s.svc", svcName, secret.Namespace), + fmt.Sprintf("%s.%s.svc.cluster.local", svcName, secret.Namespace), + } + return reconcileSignedCertWithAddresses(secret, ca, ownerRef, "system:kube-scheduler", []string{"kubernetes"}, X509UsageServerAuth, dnsNames, nil) +} + func ReconcileKubeControllerManagerClientCertSecret(secret, ca *corev1.Secret, ownerRef config.OwnerRef) error { return reconcileSignedCert(secret, ca, ownerRef, "system:kube-controller-manager", []string{"kubernetes"}, X509UsageClientAuth) } diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml index a0bb27863b2c..05a6f5cd74a2 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml @@ -26,3 +26,9 @@ status: - group: "" kind: Secret name: kube-scheduler-kubeconfig + - group: "" + kind: Service + name: kube-scheduler + - group: monitoring.coreos.com + kind: ServiceMonitor + name: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml index afe578564493..ee89332a263c 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml @@ -28,8 +28,8 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: cert-work,tmp-dir - component.hypershift.openshift.io/config-hash: 022a8a3a8c3f52a8 + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: tmp-dir + component.hypershift.openshift.io/config-hash: 022a8a3a87ecf31a87ecf31a8c3f52a8 hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64 creationTimestamp: null labels: @@ -81,6 +81,8 @@ spec: - args: - --config=/etc/kubernetes/config/config.json - --cert-dir=/var/run/kubernetes + - --tls-cert-file=/etc/kubernetes/certs/server/tls.crt + - --tls-private-key-file=/etc/kubernetes/certs/server/tls.key - --secure-port=10259 - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig @@ -201,6 +203,10 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: kube-scheduler + ports: + - containerPort: 10259 + name: client + protocol: TCP resources: requests: cpu: 25m @@ -209,12 +215,14 @@ spec: readOnlyRootFilesystem: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /var/run/kubernetes - name: cert-work - mountPath: /etc/kubernetes/kubeconfig name: kubeconfig - mountPath: /etc/kubernetes/config name: scheduler-config + - mountPath: /etc/kubernetes/certs/root-ca + name: root-ca + - mountPath: /etc/kubernetes/certs/server + name: server-crt - mountPath: /tmp name: tmp-dir initContainers: @@ -239,12 +247,26 @@ spec: operator: Equal value: hcp-namespace volumes: + - name: cluster-signer + secret: + defaultMode: 416 + secretName: cluster-signer-ca + - configMap: + defaultMode: 420 + name: root-ca + name: root-ca + - name: service-signer + secret: + defaultMode: 416 + secretName: sa-signing-key + - name: server-crt + secret: + defaultMode: 416 + secretName: kube-scheduler-server - configMap: defaultMode: 420 name: kube-scheduler name: scheduler-config - - emptyDir: {} - name: cert-work - name: kubeconfig secret: defaultMode: 416 diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml new file mode 100644 index 000000000000..e2e0307193e7 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + app: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + internalTrafficPolicy: Cluster + ipFamilyPolicy: PreferDualStack + ports: + - name: client + port: 10259 + protocol: TCP + targetPort: client + selector: + app: kube-scheduler + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml new file mode 100644 index 000000000000..d1ad0953db28 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml @@ -0,0 +1,52 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + creationTimestamp: null + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + endpoints: + - honorLabels: true + interval: 30s + metricRelabelings: + - action: keep + regex: (kube_pod_resource_request|kube_pod_resource_limit) + sourceLabels: + - __name__ + - action: replace + replacement: "" + targetLabel: _id + path: /metrics/resources + relabelings: + - action: replace + replacement: "" + targetLabel: _id + scheme: https + targetPort: client + tlsConfig: + ca: + configMap: + key: ca.crt + name: root-ca + cert: + secret: + key: tls.crt + name: kube-scheduler-server + keySecret: + key: tls.key + name: kube-scheduler-server + serverName: kube-scheduler + namespaceSelector: + matchNames: + - hcp-namespace + selector: + matchLabels: + app: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml index a0bb27863b2c..05a6f5cd74a2 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml @@ -26,3 +26,9 @@ status: - group: "" kind: Secret name: kube-scheduler-kubeconfig + - group: "" + kind: Service + name: kube-scheduler + - group: monitoring.coreos.com + kind: ServiceMonitor + name: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml index 6cc182d29759..2426ed1b74a0 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml @@ -28,8 +28,8 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: cert-work,tmp-dir - component.hypershift.openshift.io/config-hash: 022a8a3acb8019bf + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: tmp-dir + component.hypershift.openshift.io/config-hash: 022a8a3a87ecf31a87ecf31acb8019bf hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64 creationTimestamp: null labels: @@ -81,6 +81,8 @@ spec: - args: - --config=/etc/kubernetes/config/config.json - --cert-dir=/var/run/kubernetes + - --tls-cert-file=/etc/kubernetes/certs/server/tls.crt + - --tls-private-key-file=/etc/kubernetes/certs/server/tls.key - --secure-port=10259 - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig @@ -200,6 +202,10 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: kube-scheduler + ports: + - containerPort: 10259 + name: client + protocol: TCP resources: requests: cpu: 25m @@ -208,12 +214,14 @@ spec: readOnlyRootFilesystem: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /var/run/kubernetes - name: cert-work - mountPath: /etc/kubernetes/kubeconfig name: kubeconfig - mountPath: /etc/kubernetes/config name: scheduler-config + - mountPath: /etc/kubernetes/certs/root-ca + name: root-ca + - mountPath: /etc/kubernetes/certs/server + name: server-crt - mountPath: /tmp name: tmp-dir initContainers: @@ -238,12 +246,26 @@ spec: operator: Equal value: hcp-namespace volumes: + - name: cluster-signer + secret: + defaultMode: 416 + secretName: cluster-signer-ca + - configMap: + defaultMode: 420 + name: root-ca + name: root-ca + - name: service-signer + secret: + defaultMode: 416 + secretName: sa-signing-key + - name: server-crt + secret: + defaultMode: 416 + secretName: kube-scheduler-server - configMap: defaultMode: 420 name: kube-scheduler name: scheduler-config - - emptyDir: {} - name: cert-work - name: kubeconfig secret: defaultMode: 416 diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml new file mode 100644 index 000000000000..e2e0307193e7 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + app: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + internalTrafficPolicy: Cluster + ipFamilyPolicy: PreferDualStack + ports: + - name: client + port: 10259 + protocol: TCP + targetPort: client + selector: + app: kube-scheduler + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml new file mode 100644 index 000000000000..d1ad0953db28 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml @@ -0,0 +1,52 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + creationTimestamp: null + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + endpoints: + - honorLabels: true + interval: 30s + metricRelabelings: + - action: keep + regex: (kube_pod_resource_request|kube_pod_resource_limit) + sourceLabels: + - __name__ + - action: replace + replacement: "" + targetLabel: _id + path: /metrics/resources + relabelings: + - action: replace + replacement: "" + targetLabel: _id + scheme: https + targetPort: client + tlsConfig: + ca: + configMap: + key: ca.crt + name: root-ca + cert: + secret: + key: tls.crt + name: kube-scheduler-server + keySecret: + key: tls.key + name: kube-scheduler-server + serverName: kube-scheduler + namespaceSelector: + matchNames: + - hcp-namespace + selector: + matchLabels: + app: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml index a0bb27863b2c..05a6f5cd74a2 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml @@ -26,3 +26,9 @@ status: - group: "" kind: Secret name: kube-scheduler-kubeconfig + - group: "" + kind: Service + name: kube-scheduler + - group: monitoring.coreos.com + kind: ServiceMonitor + name: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml index 895eba649df8..5c9e860fae50 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml @@ -28,8 +28,8 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: cert-work,tmp-dir - component.hypershift.openshift.io/config-hash: 022a8a3acb8019bf + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: tmp-dir + component.hypershift.openshift.io/config-hash: 022a8a3a87ecf31a87ecf31acb8019bf hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64 creationTimestamp: null labels: @@ -81,6 +81,8 @@ spec: - args: - --config=/etc/kubernetes/config/config.json - --cert-dir=/var/run/kubernetes + - --tls-cert-file=/etc/kubernetes/certs/server/tls.crt + - --tls-private-key-file=/etc/kubernetes/certs/server/tls.key - --secure-port=10259 - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig @@ -201,6 +203,10 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: kube-scheduler + ports: + - containerPort: 10259 + name: client + protocol: TCP resources: requests: cpu: 25m @@ -209,12 +215,14 @@ spec: readOnlyRootFilesystem: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /var/run/kubernetes - name: cert-work - mountPath: /etc/kubernetes/kubeconfig name: kubeconfig - mountPath: /etc/kubernetes/config name: scheduler-config + - mountPath: /etc/kubernetes/certs/root-ca + name: root-ca + - mountPath: /etc/kubernetes/certs/server + name: server-crt - mountPath: /tmp name: tmp-dir initContainers: @@ -239,12 +247,26 @@ spec: operator: Equal value: hcp-namespace volumes: + - name: cluster-signer + secret: + defaultMode: 416 + secretName: cluster-signer-ca + - configMap: + defaultMode: 420 + name: root-ca + name: root-ca + - name: service-signer + secret: + defaultMode: 416 + secretName: sa-signing-key + - name: server-crt + secret: + defaultMode: 416 + secretName: kube-scheduler-server - configMap: defaultMode: 420 name: kube-scheduler name: scheduler-config - - emptyDir: {} - name: cert-work - name: kubeconfig secret: defaultMode: 416 diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml new file mode 100644 index 000000000000..e2e0307193e7 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + app: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + internalTrafficPolicy: Cluster + ipFamilyPolicy: PreferDualStack + ports: + - name: client + port: 10259 + protocol: TCP + targetPort: client + selector: + app: kube-scheduler + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml new file mode 100644 index 000000000000..d1ad0953db28 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml @@ -0,0 +1,52 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + creationTimestamp: null + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + endpoints: + - honorLabels: true + interval: 30s + metricRelabelings: + - action: keep + regex: (kube_pod_resource_request|kube_pod_resource_limit) + sourceLabels: + - __name__ + - action: replace + replacement: "" + targetLabel: _id + path: /metrics/resources + relabelings: + - action: replace + replacement: "" + targetLabel: _id + scheme: https + targetPort: client + tlsConfig: + ca: + configMap: + key: ca.crt + name: root-ca + cert: + secret: + key: tls.crt + name: kube-scheduler-server + keySecret: + key: tls.key + name: kube-scheduler-server + serverName: kube-scheduler + namespaceSelector: + matchNames: + - hcp-namespace + selector: + matchLabels: + app: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/deployment.yaml index 7e72d26c0d0e..5ccd27e206a2 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/deployment.yaml @@ -21,6 +21,8 @@ spec: - args: - --config=/etc/kubernetes/config/config.json - --cert-dir=/var/run/kubernetes + - --tls-cert-file=/etc/kubernetes/certs/server/tls.crt + - --tls-private-key-file=/etc/kubernetes/certs/server/tls.key - --secure-port=10259 - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig @@ -30,6 +32,10 @@ spec: - kube-scheduler image: hyperkube imagePullPolicy: IfNotPresent + ports: + - containerPort: 10259 + name: client + protocol: TCP livenessProbe: failureThreshold: 5 httpGet: @@ -46,19 +52,35 @@ spec: cpu: 25m memory: 150Mi volumeMounts: - - mountPath: /var/run/kubernetes - name: cert-work - mountPath: /etc/kubernetes/kubeconfig name: kubeconfig - mountPath: /etc/kubernetes/config name: scheduler-config + - mountPath: /etc/kubernetes/certs/root-ca + name: root-ca + - mountPath: /etc/kubernetes/certs/server + name: server-crt volumes: + - name: cluster-signer + secret: + defaultMode: 416 + secretName: cluster-signer-ca + - configMap: + defaultMode: 420 + name: root-ca + name: root-ca + - name: service-signer + secret: + defaultMode: 416 + secretName: sa-signing-key + - name: server-crt + secret: + defaultMode: 416 + secretName: kube-scheduler-server - configMap: defaultMode: 420 name: kube-scheduler name: scheduler-config - - emptyDir: {} - name: cert-work - name: kubeconfig secret: defaultMode: 416 diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/service.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/service.yaml new file mode 100644 index 000000000000..8226c8fde1dd --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: kube-scheduler + name: kube-scheduler +spec: + internalTrafficPolicy: Cluster + ipFamilyPolicy: PreferDualStack + ports: + - name: client + port: 10259 + protocol: TCP + targetPort: client + selector: + app: kube-scheduler + sessionAffinity: None + type: ClusterIP diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/servicemonitor.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/servicemonitor.yaml new file mode 100644 index 000000000000..805c152d1595 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,36 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kube-scheduler + namespace: HCP_NAMESPACE +spec: + endpoints: + - metricRelabelings: + - action: keep + regex: (kube_pod_resource_request|kube_pod_resource_limit) + sourceLabels: + - __name__ + path: /metrics/resources + honorLabels: true + interval: 30s + scheme: https + targetPort: client + tlsConfig: + ca: + configMap: + key: ca.crt + name: root-ca + cert: + secret: + key: tls.crt + name: kube-scheduler-server + keySecret: + key: tls.key + name: kube-scheduler-server + serverName: kube-scheduler + namespaceSelector: + matchNames: + - HCP_NAMESPACE + selector: + matchLabels: + app: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component.go index f385e733822d..47e2dbf4cfef 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component.go @@ -40,6 +40,13 @@ func NewComponent() component.ControlPlaneComponent { "kubeconfig.yaml", component.WithAdaptFunction(adaptKubeconfig), ). + WithManifestAdapter( + "servicemonitor.yaml", + component.WithAdaptFunction(adaptServiceMonitor), + ). + WithManifestAdapter( + "service.yaml", + ). InjectAvailabilityProberContainer(util.AvailabilityProberOpts{}). Build() } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor.go new file mode 100644 index 000000000000..9370b5205121 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor.go @@ -0,0 +1,20 @@ +package scheduler + +import ( + _ "embed" + + component "github.com/openshift/hypershift/support/controlplane-component" + "github.com/openshift/hypershift/support/util" + + prometheusoperatorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" +) + +func adaptServiceMonitor(cpContext component.WorkloadContext, sm *prometheusoperatorv1.ServiceMonitor) error { + sm.Spec.NamespaceSelector = prometheusoperatorv1.NamespaceSelector{ + MatchNames: []string{cpContext.HCP.Namespace}, + } + for i := range len(sm.Spec.Endpoints) { + util.ApplyClusterIDLabel(&sm.Spec.Endpoints[i], cpContext.HCP.Spec.ClusterID) + } + return nil +}