diff --git a/charts/base-cluster/ci/artifacthub-values-3.yaml b/charts/base-cluster/ci/artifacthub-values-3.yaml new file mode 100644 index 0000000000..0d0edd8651 --- /dev/null +++ b/charts/base-cluster/ci/artifacthub-values-3.yaml @@ -0,0 +1,7 @@ +global: + clusterName: test + baseDomain: example.com + serviceLevelAgreement: None +backup: + provider: + k8up: {} diff --git a/charts/base-cluster/ci/artifacthub-values.yaml b/charts/base-cluster/ci/artifacthub-values.yaml index 720186d244..50916299c6 100644 --- a/charts/base-cluster/ci/artifacthub-values.yaml +++ b/charts/base-cluster/ci/artifacthub-values.yaml @@ -7,16 +7,18 @@ flux: kyverno: enabled: true backup: - defaultLocation: location - backupStorageLocations: - location: - provider: - minio: - url: https://minio.com - existingSecret: - name: secret - prefix: /prefix - bucket: bucket + provider: + velero: + defaultLocation: location + backupStorageLocations: + location: + provider: + minio: + url: https://minio.com + existingSecret: + name: secret + prefix: /prefix + bucket: bucket storage: readWriteMany: enabled: true diff --git a/charts/base-cluster/ci/velero-backupStorageLocations-gen-values.sh b/charts/base-cluster/ci/velero-backupStorageLocations-gen-values.sh index 80839620f1..3a7d80184a 100755 --- a/charts/base-cluster/ci/velero-backupStorageLocations-gen-values.sh +++ b/charts/base-cluster/ci/velero-backupStorageLocations-gen-values.sh @@ -16,4 +16,4 @@ for prefix in 'prefix="/prod"' ''; do yq -y -n "{test$((i++)): ({} | .provider.$provider = ({} | .$url | .$auth | .$existingSecret) | .$prefix | .$bucket)}" done done -done | yq -s '{backup: {backupStorageLocations: .[], defaultLocation: "test0"}}' +done | yq -s '{backup: { provider: { velero: { backupStorageLocations: .[], defaultLocation: "test0"}}}}' diff --git a/charts/base-cluster/templates/backup/k8up/k8up.yaml b/charts/base-cluster/templates/backup/k8up/k8up.yaml new file mode 100644 index 0000000000..ec9ba0c7a6 --- /dev/null +++ b/charts/base-cluster/templates/backup/k8up/k8up.yaml @@ -0,0 +1,61 @@ +{{- if ne (.Values.backup.provider).k8up nil -}} +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: k8up + namespace: backup + labels: {{- include "common.labels.standard" $ | nindent 4 }} + app.kubernetes.io/component: backup +spec: + chart: + spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "k8up" "chart" "k8up" "context" $) | nindent 6 }} + interval: 1h + driftDetection: + mode: enabled + {{- if .Values.monitoring.prometheus.enabled }} + dependsOn: + - name: kube-prometheus-stack + namespace: monitoring + {{- end }} + install: + crds: CreateReplace + upgrade: + crds: CreateReplace + values: + {{- with .Values.global.imageRegistry }} + image: + registry: {{ . }} + {{- end }} + podSecurityContext: + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + securityContext: + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + resources: {{- include "common.resources" .Values.backup | nindent 6 }} + k8up: + globalResources: {{- include "common.resources" .Values.backup.runners | nindent 8 }} + skipWithoutAnnotation: true + priorityClassName: system-cluster-critical + {{- if .Values.monitoring.prometheus.enabled }} + metrics: + serviceMonitor: + additionalLabels: + monitoring/provisioned-by: base-cluster + enabled: true + prometheusRule: + createDefaultRules: true + grafanaDashboard: + enabled: true + additionalLabels: + grafana_dashboard: "1" + {{- end }} +{{- end -}} diff --git a/charts/base-cluster/templates/backup/migrations/velero-4-to-5.yaml b/charts/base-cluster/templates/backup/migrations/velero-4-to-5.yaml deleted file mode 100644 index ab33c0f2ed..0000000000 --- a/charts/base-cluster/templates/backup/migrations/velero-4-to-5.yaml +++ /dev/null @@ -1,93 +0,0 @@ -{{- if and .Values.backup.backupStorageLocations (hasPrefix "4." (dig "spec" "chart" "spec" "version" "" (lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" "backup" "velero"))) }} -apiVersion: batch/v1 -kind: Job -metadata: - name: remove-backup-storage-locations - namespace: backup - labels: {{- include "common.labels.standard" . | nindent 4 }} - annotations: - helm.sh/hook: pre-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - backoffLimit: 0 - template: - spec: - restartPolicy: Never - automountServiceAccountToken: true - serviceAccountName: remove-backup-storage-locations - securityContext: - runAsNonRoot: true - runAsGroup: 1000 - runAsUser: 1000 - fsGroup: 1000 - containers: - - name: remove-backup-storage-locations - image: {{ include "common.images.image" (dict "imageRoot" .Values.global.kubectl.image "global" .Values.global) }} - imagePullPolicy: {{ include "common.images.pullPolicy" .Values.global.kubectl.image }} - securityContext: - readOnlyRootFilesystem: true - privileged: false - capabilities: - drop: - - ALL - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault - command: - - bash - - -ex - - -c - - | - kubectl get backupstoragelocations # test if RBAC is working - for NAME in {{ .Values.backup.backupStorageLocations | keys | join " " }}; do - if kubectl get backupstoragelocation $NAME >/dev/null; then - kubectl delete backupstoragelocation $NAME - fi - done ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: remove-backup-storage-locations - namespace: backup - labels: {{- include "common.labels.standard" . | nindent 4 }} - annotations: - helm.sh/hook: pre-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: remove-backup-storage-locations - namespace: backup - labels: {{- include "common.labels.standard" . | nindent 4 }} - annotations: - helm.sh/hook: pre-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -rules: - - verbs: - - list - - get - - delete - resources: - - backupstoragelocations - apiGroups: - - velero.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: remove-backup-storage-locations - namespace: backup - labels: {{- include "common.labels.standard" . | nindent 4 }} - annotations: - helm.sh/hook: pre-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -subjects: - - kind: ServiceAccount - name: remove-backup-storage-locations -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: remove-backup-storage-locations -{{- end }} diff --git a/charts/base-cluster/templates/backup/validation.yaml b/charts/base-cluster/templates/backup/validation.yaml deleted file mode 100644 index 5d95e90ac5..0000000000 --- a/charts/base-cluster/templates/backup/validation.yaml +++ /dev/null @@ -1,5 +0,0 @@ -{{- if .Values.backup.defaultLocation -}} - {{- if not (index .Values.backup.backupStorageLocations .Values.backup.defaultLocation) -}} - {{- fail (printf "The `defaultLocation` '%s' must exist in `backupStorageLocations`" .Values.backup.defaultLocation) -}} - {{- end -}} -{{- end -}} diff --git a/charts/base-cluster/templates/backup/_helpers.tpl b/charts/base-cluster/templates/backup/velero/_helpers.tpl similarity index 66% rename from charts/base-cluster/templates/backup/_helpers.tpl rename to charts/base-cluster/templates/backup/velero/_helpers.tpl index de4fa5076f..3b1d3f655b 100644 --- a/charts/base-cluster/templates/backup/_helpers.tpl +++ b/charts/base-cluster/templates/backup/velero/_helpers.tpl @@ -1,4 +1,4 @@ -{{- define "base-cluster.backup.getProviderName" -}} +{{- define "base-cluster.backup.velero.getProviderName" -}} {{- $providers := list "minio" -}} {{- $providerName := . | keys | first -}} {{- if has $providerName $providers -}} @@ -8,14 +8,14 @@ {{- end -}} {{- end -}} -{{- define "base-cluster.backup.mapProviderName" -}} +{{- define "base-cluster.backup.velero.mapProviderName" -}} {{- $providerMap := dict "minio" "aws" -}} {{- get $providerMap . | required "Missing provider mapping" -}} {{- end -}} -{{- define "base-cluster.backup.credential" -}} - {{- $providerName := include "base-cluster.backup.getProviderName" . }} - {{- $pluginName := include "base-cluster.backup.mapProviderName" $providerName -}} +{{- define "base-cluster.backup.velero.credential" -}} + {{- $providerName := include "base-cluster.backup.velero.getProviderName" . }} + {{- $pluginName := include "base-cluster.backup.velero.mapProviderName" $providerName -}} {{- $provider := get . $providerName }} {{- if eq $pluginName "aws" -}} {{- if hasKey $provider "accessKeyID" -}} @@ -28,9 +28,9 @@ aws_secret_access_key={{ get $provider "secretAccessKey" }} {{- end -}} {{- end -}} -{{- define "base-cluster.backup.credentialType" -}} - {{- $providerName := include "base-cluster.backup.getProviderName" . }} - {{- $pluginName := include "base-cluster.backup.mapProviderName" $providerName -}} +{{- define "base-cluster.backup.velero.credentialType" -}} + {{- $providerName := include "base-cluster.backup.velero.getProviderName" . }} + {{- $pluginName := include "base-cluster.backup.velero.mapProviderName" $providerName -}} {{- $provider := get . $providerName -}} {{- if hasKey $provider "existingSecret" -}} existingSecret diff --git a/charts/base-cluster/templates/backup/credentials.yaml b/charts/base-cluster/templates/backup/velero/credentials.yaml similarity index 70% rename from charts/base-cluster/templates/backup/credentials.yaml rename to charts/base-cluster/templates/backup/velero/credentials.yaml index e6eddf3bd5..1b2b2284bf 100644 --- a/charts/base-cluster/templates/backup/credentials.yaml +++ b/charts/base-cluster/templates/backup/velero/credentials.yaml @@ -1,7 +1,7 @@ -{{- if .Values.backup.backupStorageLocations }} +{{- if ((.Values.backup.provider).velero).backupStorageLocations }} {{- $providerMap := dict "minio" "accessKeyID" -}} {{- range $name, $spec := .Values.backup.backupStorageLocations -}} - {{- $providerName := include "base-cluster.backup.getProviderName" $spec.provider -}} + {{- $providerName := include "base-cluster.backup.velero.getProviderName" $spec.provider -}} {{- if dig $providerName (get $providerMap $providerName | required (printf "Credentials for provider '%s' not implemented" $providerName)) false $spec.provider }} --- apiVersion: v1 @@ -13,7 +13,7 @@ metadata: app.kubernetes.io/component: {{ $name }} app.kubernetes.io/part-of: backup stringData: - {{ $providerName }}: |-{{- include "base-cluster.backup.credential" $spec.provider | nindent 4 }} + {{ $providerName }}: |-{{- include "base-cluster.backup.velero.credential" $spec.provider | nindent 4 }} {{- end -}} {{- end }} {{- end }} diff --git a/charts/base-cluster/templates/backup/velero/validation.yaml b/charts/base-cluster/templates/backup/velero/validation.yaml new file mode 100644 index 0000000000..efda9191c1 --- /dev/null +++ b/charts/base-cluster/templates/backup/velero/validation.yaml @@ -0,0 +1,5 @@ +{{- if ((.Values.backup.provider).velero).defaultLocation -}} + {{- if not (index .Values.backup.provider.velero.backupStorageLocations .Values.backup.provider.velero.defaultLocation) -}} + {{- fail (printf "The `defaultLocation` '%s' must exist in `backupStorageLocations`" .Values.backup.provider.velero.defaultLocation) -}} + {{- end -}} +{{- end -}} diff --git a/charts/base-cluster/templates/backup/velero.yaml b/charts/base-cluster/templates/backup/velero/velero.yaml similarity index 85% rename from charts/base-cluster/templates/backup/velero.yaml rename to charts/base-cluster/templates/backup/velero/velero.yaml index 7722ae19e1..60b6298101 100644 --- a/charts/base-cluster/templates/backup/velero.yaml +++ b/charts/base-cluster/templates/backup/velero/velero.yaml @@ -1,4 +1,4 @@ -{{- if .Values.backup.backupStorageLocations }} +{{- if (.Values.backup.provider).velero }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -55,20 +55,20 @@ spec: priorityClassName: system-cluster-critical deployNodeAgent: true # enable FSB nodeAgent: - resources: {{- include "common.resources" .Values.backup.nodeAgent | nindent 8 }} + resources: {{- include "common.resources" .Values.backup.runners | nindent 8 }} priorityClassName: system-cluster-critical configuration: - defaultBackupStorageLocation: {{ .Values.backup.defaultLocation | quote }} - backupStorageLocation: {{- range $name, $spec := .Values.backup.backupStorageLocations }} - {{- $providerName := include "base-cluster.backup.getProviderName" $spec.provider }} + defaultBackupStorageLocation: {{ .Values.backup.provider.velero.defaultLocation | quote }} + backupStorageLocation: {{- range $name, $spec := .Values.backup.provider.velero.backupStorageLocations }} + {{- $providerName := include "base-cluster.backup.velero.getProviderName" $spec.provider }} - name: {{ $name | quote }} - provider: {{ include "base-cluster.backup.mapProviderName" $providerName | quote }} + provider: {{ include "base-cluster.backup.velero.mapProviderName" $providerName | quote }} bucket: {{ $spec.bucket | quote }} {{- if $spec.prefix }} prefix: {{ $spec.prefix | quote }} {{- end }} - default: {{ eq $name $.Values.backup.defaultLocation }} - {{- $credentialType := include "base-cluster.backup.credentialType" $spec.provider }} + default: {{ eq $name $.Values.backup.provider.velero.defaultLocation }} + {{- $credentialType := include "base-cluster.backup.velero.credentialType" $spec.provider }} {{- if eq $credentialType "direct" }} credential: name: {{ printf "%s-velero-backuplocation-%s" (include "common.names.fullname" $) $name }} @@ -105,5 +105,4 @@ spec: labels: severity: critical period: WorkingHours - {{- end }} diff --git a/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_grafana-config.yaml b/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_grafana-config.yaml index 907fe64666..d154734517 100644 --- a/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_grafana-config.yaml +++ b/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_grafana-config.yaml @@ -171,12 +171,18 @@ dashboards: gnetId: 17813 revision: 2 {{- end }} - {{- if .Values.backup.backupStorageLocations }} + {{- if (.Values.backup.provider).velero }} velero: <<: *dashboard gnetId: 11055 revision: 2 {{- end }} + {{- if ne (.Values.backup.provider).k8up nil }} + k8up: + <<: *dashboard + gnetId: 20166 + revision: 4 + {{- end }} {{- with .Values.monitoring.grafana.additionalDashboards }} {{ toYaml . | nindent 4 }} {{- end }} diff --git a/charts/base-cluster/values.schema.json b/charts/base-cluster/values.schema.json index 83115a9025..087747c46e 100644 --- a/charts/base-cluster/values.schema.json +++ b/charts/base-cluster/values.schema.json @@ -1133,7 +1133,8 @@ }, "required": [ "cloudflare" - ] + ], + "additionalProperties": false }, { "type": "null" @@ -1542,118 +1543,166 @@ "resources": { "$ref": "#/$defs/resourceRequirements" }, - "backupStorageLocations": { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "provider": { - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "minio": { + "provider": { + "oneOf": [ + { + "type": "object", + "properties": { + "velero": { + "type": "object", + "properties": { + "backupStorageLocations": { + "type": "object", + "additionalProperties": { "type": "object", - "oneOf": [ - { - "required": [ - "accessKeyID", - "secretAccessKey" - ] - }, - { - "required": [ - "existingSecret" - ] - }, - { - "not": { - "anyOf": [ - { - "required": [ - "accessKeyID" - ] - }, - { - "required": [ - "secretAccessKey" - ] - }, - { - "required": [ - "existingSecret" - ] - } - ] - } - } - ], "properties": { - "accessKeyID": { - "type": "string" - }, - "secretAccessKey": { - "type": "string" - }, - "existingSecret": { + "provider": { "type": "object", - "properties": { - "name": { - "type": "string" - }, - "key": { - "type": "string", - "description": "The default is <$providerName-$name> (e.g. 'minio-prod')" + "oneOf": [ + { + "type": "object", + "properties": { + "minio": { + "type": "object", + "oneOf": [ + { + "required": [ + "accessKeyID", + "secretAccessKey" + ] + }, + { + "required": [ + "existingSecret" + ] + }, + { + "not": { + "anyOf": [ + { + "required": [ + "accessKeyID" + ] + }, + { + "required": [ + "secretAccessKey" + ] + }, + { + "required": [ + "existingSecret" + ] + } + ] + } + } + ], + "properties": { + "accessKeyID": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + }, + "existingSecret": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "key": { + "type": "string", + "description": "The default is <$providerName-$name> (e.g. 'minio-prod')" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + "url": { + "type": "string" + }, + "region": { + "type": "string", + "default": "Region1" + }, + "forcePathStyle": { + "type": "boolean", + "default": true + } + }, + "required": [ + "url" + ], + "additionalProperties": false + } + }, + "additionalProperties": false } - }, - "required": [ - "name" ], - "additionalProperties": false + "minProperties": 1, + "maxProperties": 1 }, - "url": { + "bucket": { "type": "string" }, - "region": { - "type": "string", - "default": "Region1" - }, - "forcePathStyle": { - "type": "boolean", - "default": true + "prefix": { + "type": "string" } }, "required": [ - "url" + "provider", + "bucket" ], "additionalProperties": false } }, - "additionalProperties": false - } - ], - "minProperties": 1, - "maxProperties": 1 + "defaultLocation": { + "type": "string" + } + }, + "oneOf": [ + { + "properties": { + "backupStorageLocations": { + "minProperties": 1 + } + }, + "required": [ + "defaultLocation" + ] + }, + { + "properties": { + "backupStorageLocations": { + "maxProperties": 0 + } + } + } + ], + "additionalProperties": false + } }, - "bucket": { - "type": "string" + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "k8up": { + "type": "object", + "additionalProperties": false + } }, - "prefix": { - "type": "string" - } + "additionalProperties": false }, - "required": [ - "provider", - "bucket" - ], - "additionalProperties": false - } - }, - "defaultLocation": { - "type": "string" + { + "type": "null" + } + ] }, - "nodeAgent": { + "runners": { "type": "object", "properties": { "resourcesPreset": { @@ -1666,25 +1715,6 @@ "additionalProperties": false } }, - "oneOf": [ - { - "properties": { - "backupStorageLocations": { - "minProperties": 1 - } - }, - "required": [ - "defaultLocation" - ] - }, - { - "properties": { - "backupStorageLocations": { - "maxProperties": 0 - } - } - } - ], "additionalProperties": false }, "kube-janitor": { diff --git a/charts/base-cluster/values.yaml b/charts/base-cluster/values.yaml index 0a09a8f54f..9e55121d4f 100644 --- a/charts/base-cluster/values.yaml +++ b/charts/base-cluster/values.yaml @@ -51,7 +51,7 @@ global: app.kubernetes.io/component: storage app.kubernetes.io/part-of: nfs-server-provisioner backup: - condition: "{{ not (empty .Values.backup.backupStorageLocations) }}" + condition: "{{ not (empty .Values.backup.provider) }}" additionalLabels: app.kubernetes.io/component: backup certificates: @@ -183,6 +183,11 @@ global: charts: reflector: 9.1.22 condition: '{{ include "base-cluster.reflector.enabled" (dict "context" .) }}' + k8up: + url: https://k8up-io.github.io/k8up + charts: + k8up: 4.8.6 + condition: '{{ ne (.Values.backup.provider).k8up nil }}' vmware: url: https://vmware-tanzu.github.io/helm-charts charts: @@ -452,7 +457,7 @@ rbac: accounts: {} backup: - backupStorageLocations: {} + provider: null resources: requests: cpu: 5m @@ -460,7 +465,7 @@ backup: limits: cpu: 500m memory: 500Mi - nodeAgent: + runners: resources: requests: cpu: 5m