diff --git a/charts/t8s-cluster/templates/management-cluster/autoscaler.yaml b/charts/t8s-cluster/templates/management-cluster/autoscaler.yaml new file mode 100644 index 0000000000..b2026ab902 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/autoscaler.yaml @@ -0,0 +1,35 @@ +{{- $enabled := false -}} +{{- range $_, $machineDeploymentClass := .Values.nodePools }} + {{- with $machineDeploymentClass.replicas }} + {{- $enabled = or $enabled (kindIs "map" .) -}} + {{- end }} +{{- end }} +{{- if $enabled -}} +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: {{ printf "%s-autoscaler" .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: {{- include "common.labels.standard" . | nindent 4 }} +spec: + chart: + spec: {{- include "common.helm.chartSpec" (dict "repo" "autoscaler" "repoNamespace" "flux-system" "chart" "cluster-autoscaler" "context" $) | nindent 6 }} + interval: 1h + driftDetection: + mode: enabled + values: + autoDiscovery: + clusterName: {{ .Release.Name }} + cloudProvider: clusterapi + # workload-management + clusterAPIMode: kubeconfig-incluster + clusterAPIConfigMapsNamespace: {{ .Release.Namespace }} + clusterAPIKubeconfigSecret: {{ .Release.Name }}-kubeconfig + extraArgs: + scale-down-utilization-threshold: 0.8 + leader-elect-resource-name: {{ .Release.Name }}-cluster-autoscaler + status-config-map-name: {{ .Release.Name }}-cluster-autoscaler + serviceMonitor: + enabled: true + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/charts/t8s-cluster/templates/workload-cluster/pre-install/_uninstall-job.yaml b/charts/t8s-cluster/templates/workload-cluster/pre-install/_uninstall-job.yaml deleted file mode 100644 index 7f71bd1e6c..0000000000 --- a/charts/t8s-cluster/templates/workload-cluster/pre-install/_uninstall-job.yaml +++ /dev/null @@ -1,140 +0,0 @@ -{{- define "t8s-cluster.workload.uninstall-job" -}} - {{- $name := .name -}} - {{- $clusterResourceSet := .clusterResourceSet -}} - {{- $_ := mustMerge . (pick .context "Values" "Release" "Chart") -}} - {{- if lookup "addons.cluster.x-k8s.io/v1beta1" "ClusterResourceSet" .Release.Namespace $clusterResourceSet -}} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ $name }}-uninstall - namespace: {{ .Release.Namespace }} - labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: {{ $name }} - app.kubernetes.io/part-of: pre-install-upgrade-cleanup - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "1" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - template: - spec: - restartPolicy: Never - securityContext: - runAsNonRoot: true - runAsGroup: 1000 - runAsUser: 1000 - fsGroup: 1000 - serviceAccountName: {{ $name }}-uninstall - enableServiceLinks: false - containers: - - name: uninstall - image: {{ include "common.images.image" (dict "imageRoot" .Values.global.kubectl.image "global" .Values.global) }} - imagePullPolicy: {{ include "common.images.pullPolicy" .Values.global.kubectl.image }} - securityContext: - readOnlyRootFilesystem: true - privileged: false - capabilities: - drop: - - ALL - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /tmp/workload-kubeconfig - name: workload-kubeconfig - subPath: value - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CLUSTERRESOURCESET - value: {{ $clusterResourceSet | quote }} - command: - - bash - - -ex - - -c - - |- - set -o pipefail - {{- if .Release.IsInstall }} - for resource in $(kubectl -n "$NAMESPACE" get clusterresourceset "$CLUSTERRESOURCESET" -o json | jq -r -c '.spec.resources | map({kind: .kind, name: .name})[]'); do - kind="$(jq -r .kind <<<"$resource")" - name="$(jq -r .name <<<"$resource")" - case "$kind" in - ConfigMap) - kubectl -n "$NAMESPACE" get configmap "$name" -o json | jq -r '.data[]' - ;; - Secret) - secret="$(kubectl -n "$NAMESPACE" get secret "$name" -o json)" - for key in $(jq -r '.data | keys[]' <<<"$secret"); do - jq -r --arg key "$key" '.data[$key]' <<<"$secret" | base64 -d - echo '---' - done - ;; - esac | kubectl --kubeconfig /tmp/workload-kubeconfig delete -f - - kubectl -n "$NAMESPACE" delete "$kind" "$name" - done - {{- end }} - kubectl -n "$NAMESPACE" delete clusterresourceset "$CLUSTERRESOURCESET" - volumes: - - name: workload-kubeconfig - secret: - secretName: {{ .Release.Name }}-kubeconfig - optional: false ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ $name }}-uninstall - namespace: {{ .Release.Namespace }} - labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: {{ $name }} - app.kubernetes.io/part-of: pre-install-cleanup - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ $name }}-uninstall - namespace: {{ .Release.Namespace }} - labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: {{ $name }} - app.kubernetes.io/part-of: pre-install-cleanup - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -rules: - - verbs: - - get - - delete - - list - resources: - - secrets - - configmaps - - clusterresourcesets - apiGroups: - - "" - - addons.cluster.x-k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ $name }}-uninstall - namespace: {{ .Release.Namespace }} - labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: {{ $name }} - app.kubernetes.io/part-of: pre-install-cleanup - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -subjects: - - kind: ServiceAccount - name: {{ $name }}-uninstall -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ $name }}-uninstall - {{- end -}} -{{- end -}} diff --git a/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-cloud-controller-manager.yaml b/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-cloud-controller-manager.yaml deleted file mode 100644 index 0b916e1c53..0000000000 --- a/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-cloud-controller-manager.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ include "t8s-cluster.workload.uninstall-job" (dict "name" "cloud-controller-manager" "context" . "clusterResourceSet" "cloud-controller-manager") }} diff --git a/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-cni.yaml b/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-cni.yaml deleted file mode 100644 index 4d2086d2a9..0000000000 --- a/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-cni.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "t8s-cluster.cni" .) "cilium" -}} -{{ include "t8s-cluster.workload.uninstall-job" (dict "name" "cni" "context" . "clusterResourceSet" "cni") }} -{{- end -}} diff --git a/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-csi.yaml b/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-csi.yaml deleted file mode 100644 index 0ee5769d74..0000000000 --- a/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-csi.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ include "t8s-cluster.workload.uninstall-job" (dict "name" "csi" "context" . "clusterResourceSet" "csi") }} diff --git a/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-etcd-defrag.yaml b/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-etcd-defrag.yaml deleted file mode 100644 index 657e2ae9a3..0000000000 --- a/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-etcd-defrag.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ include "t8s-cluster.workload.uninstall-job" (dict "name" "etcd-defrag" "context" . "clusterResourceSet" "kube-etcd-defrag") }} diff --git a/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-storageclasses.yaml b/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-storageclasses.yaml deleted file mode 100644 index a394e5ed2f..0000000000 --- a/charts/t8s-cluster/templates/workload-cluster/pre-install/uninstall-storageclasses.yaml +++ /dev/null @@ -1,60 +0,0 @@ -{{- if lookup "v1" "Secret" .Release.Namespace (printf "%s-kubeconfig" .Release.Name) -}} -apiVersion: batch/v1 -kind: Job -metadata: - name: storageclasses-uninstall - namespace: {{ .Release.Namespace }} - labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: storageclasses - app.kubernetes.io/part-of: pre-install-cleanup - annotations: - helm.sh/hook: pre-install - helm.sh/hook-weight: "1" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - template: - spec: - restartPolicy: Never - securityContext: - runAsNonRoot: true - runAsGroup: 1000 - runAsUser: 1000 - fsGroup: 1000 - automountServiceAccountToken: false - enableServiceLinks: false - containers: - - name: uninstall - image: {{ include "common.images.image" (dict "imageRoot" .Values.global.kubectl.image "global" .Values.global) }} - imagePullPolicy: {{ include "common.images.pullPolicy" .Values.global.kubectl.image }} - securityContext: - readOnlyRootFilesystem: true - privileged: false - capabilities: - drop: - - ALL - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault - env: - - name: KUBECONFIG - value: /tmp/workload-kubeconfig - volumeMounts: - - mountPath: /tmp/workload-kubeconfig - name: workload-kubeconfig - subPath: value - command: - - bash - - -ex - - -c - - |- - for storageclass in teutostack-ssd teutostack-hdd; do - if kubectl get storageclass "$storageclass" >/dev/null; then - kubectl delete storageclass "$storageclass" - fi - done - volumes: - - name: workload-kubeconfig - secret: - secretName: {{ .Release.Name }}-kubeconfig - optional: false -{{- end }} diff --git a/charts/t8s-cluster/values.yaml b/charts/t8s-cluster/values.yaml index 9d14299914..0f541f0a3e 100644 --- a/charts/t8s-cluster/values.yaml +++ b/charts/t8s-cluster/values.yaml @@ -26,6 +26,10 @@ global: openstack-cinder-csi 2.29.x: 2.29.2 openstack-cinder-csi 2.28.x: 2.28.3 openstack-cinder-csi 2.27.x: 2.27.3 + autoscaler: + url: https://kubernetes.github.io/autoscaler + charts: + cluster-autoscaler: 9.52.1 cetic: url: https://cetic.github.io/helm-charts etcd: