diff --git a/deployment/clouddeploy/gke-workers/base/ksm_cluster_pod_monitoring.yaml b/deployment/clouddeploy/gke-workers/base/ksm_cluster_pod_monitoring.yaml new file mode 100644 index 00000000000..54fff10588f --- /dev/null +++ b/deployment/clouddeploy/gke-workers/base/ksm_cluster_pod_monitoring.yaml @@ -0,0 +1,22 @@ +apiVersion: monitoring.googleapis.com/v1 +kind: ClusterPodMonitoring +metadata: + name: kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: google-cloud-managed-prometheus +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + endpoints: + - port: metrics + # Customized from 30s to 5s. + interval: 5s + metricRelabeling: + - action: keep + # Curated subset of metrics to reduce costs, currently just targeting cronjob and pod resources. + regex: kube_(cronjob|pod)(_.+)? + sourceLabels: [__name__] + targetLabels: + metadata: [] # explicitly empty so the metric labels are respected diff --git a/deployment/clouddeploy/gke-workers/base/ksm_cluster_role.yaml b/deployment/clouddeploy/gke-workers/base/ksm_cluster_role.yaml new file mode 100644 index 00000000000..d0fa29b012d --- /dev/null +++ b/deployment/clouddeploy/gke-workers/base/ksm_cluster_role.yaml @@ -0,0 +1,130 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gmp-public:kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.14.0 +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + - nodes + - pods + - services + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - extensions + resources: + - daemonsets + - deployments + - replicasets + - ingresses + verbs: + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - list + - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - list + - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - volumeattachments + verbs: + - list + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + - ingresses + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch diff --git a/deployment/clouddeploy/gke-workers/base/ksm_cluster_role_binding.yaml b/deployment/clouddeploy/gke-workers/base/ksm_cluster_role_binding.yaml new file mode 100644 index 00000000000..bc403a2a102 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/base/ksm_cluster_role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gmp-public:kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.14.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gmp-public:kube-state-metrics +subjects: +- kind: ServiceAccount + namespace: gmp-public + name: kube-state-metrics diff --git a/deployment/clouddeploy/gke-workers/base/ksm_horizontal_pod_autoscaler.yaml b/deployment/clouddeploy/gke-workers/base/ksm_horizontal_pod_autoscaler.yaml new file mode 100644 index 00000000000..dfaa5384243 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/base/ksm_horizontal_pod_autoscaler.yaml @@ -0,0 +1,29 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: kube-state-metrics + namespace: gmp-public +spec: + maxReplicas: 10 + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: kube-state-metrics + metrics: + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 60 + behavior: + scaleDown: + policies: + - type: Pods + value: 1 + # Under-utilization needs to persist for `periodSeconds` before any action can be taken. + # Current supported max from https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/horizontal-pod-autoscaler-v2beta2/. + periodSeconds: 1800 + # Current supported max from https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/horizontal-pod-autoscaler-v2beta2/. + stabilizationWindowSeconds: 3600 diff --git a/deployment/clouddeploy/gke-workers/base/ksm_pod_monitoring.yaml b/deployment/clouddeploy/gke-workers/base/ksm_pod_monitoring.yaml new file mode 100644 index 00000000000..1478b0f81b2 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/base/ksm_pod_monitoring.yaml @@ -0,0 +1,15 @@ +apiVersion: monitoring.googleapis.com/v1 +kind: PodMonitoring +metadata: + namespace: gmp-public + name: kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: google-cloud-managed-prometheus +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + endpoints: + - port: metrics-self + interval: 30s diff --git a/deployment/clouddeploy/gke-workers/base/ksm_service.yaml b/deployment/clouddeploy/gke-workers/base/ksm_service.yaml new file mode 100644 index 00000000000..784ceb569b5 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/base/ksm_service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.14.0 + namespace: gmp-public + name: kube-state-metrics +spec: + clusterIP: None + ports: + - name: metrics + port: 8080 + targetPort: metrics + - name: metrics-self + port: 8081 + targetPort: metrics-self + selector: + app.kubernetes.io/name: kube-state-metrics diff --git a/deployment/clouddeploy/gke-workers/base/ksm_service_account.yaml b/deployment/clouddeploy/gke-workers/base/ksm_service_account.yaml new file mode 100644 index 00000000000..3b08242f934 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/base/ksm_service_account.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: gmp-public + name: kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.14.0 diff --git a/deployment/clouddeploy/gke-workers/base/ksm_stateful_set.yaml b/deployment/clouddeploy/gke-workers/base/ksm_stateful_set.yaml new file mode 100644 index 00000000000..e098991f198 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/base/ksm_stateful_set.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.14.0 + namespace: gmp-public + name: kube-state-metrics +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + serviceName: kube-state-metrics + template: + metadata: + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.14.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - amd64 + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - name: kube-state-metric + image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.14.0 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + - --port=8080 + - --telemetry-port=8081 + ports: + - name: metrics + containerPort: 8080 + - name: metrics-self + containerPort: 8081 + resources: + requests: + cpu: 100m + memory: 190Mi + limits: + memory: 250Mi + securityContext: + allowPrivilegeEscalation: false + privileged: false + capabilities: + drop: + - all + runAsUser: 1000 + runAsGroup: 1000 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 8081 + initialDelaySeconds: 5 + timeoutSeconds: 5 + serviceAccountName: kube-state-metrics diff --git a/deployment/clouddeploy/gke-workers/base/kustomization.yaml b/deployment/clouddeploy/gke-workers/base/kustomization.yaml index 293e8e0b943..48efdbfee68 100644 --- a/deployment/clouddeploy/gke-workers/base/kustomization.yaml +++ b/deployment/clouddeploy/gke-workers/base/kustomization.yaml @@ -16,3 +16,11 @@ resources: - nvd-mirror.yaml - backup.yaml - generate-sitemap.yaml +- ksm_cluster_pod_monitoring.yaml +- ksm_cluster_role_binding.yaml +- ksm_cluster_role.yaml +- ksm_horizontal_pod_autoscaler.yaml +- ksm_pod_monitoring.yaml +- ksm_service_account.yaml +- ksm_service.yaml +- ksm_stateful_set.yaml