Skip to content

Commit befcbbb

Browse files
stephenmoloneystephenmoloney
authored
Added: Backup capabilities (#4)
* Added: Backup capabilities ***What does this change do?*** - Added ability to execute backups as a standalone job - Added ability to execute backups as a cronjob - Backups leverage existing technology in [RESTIC](https://restic.net/) and a S3 compliant object storage backend - Ability to backup the postgresql DB and ability to backup the file system with data files - Some minor fixes for default handling of the ingresses in env configmap - Added ENV variables specific to RESTIC backups - Added a backup section in `values.yaml` for the helm values file to set the required configuration for the S3 compliant backend - Minor improvements to CI - Added a bash function or file which allows for the templating of a single file - This is used for `execute backups as a standalone job` ***Why is this change needed?*** - Facilitate backups after saleor has been deployed * Fixed: version libssl1.1 dep version * Fixed: version libcairo2 dep version * Fixed: version of libmagic1 * Fixed: issue with postgresql client version
1 parent bd6749e commit befcbbb

22 files changed

+639
-97
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ saleor_storefront
88
charts/saleor-platform/charts
99
charts/saleor-platform/tmpcharts
1010
charts/saleor-platform/Chart.lock
11+
charts/saleor-platform/ChartBak.yaml
12+
chart-releaser_*
1113
downloads
1214
*-x86_64
1315
index.yaml

‎charts/saleor-core/Chart.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ apiVersion: v2
44
name: saleor-core
55
description: A Helm chart for deploying the saleor core application in Kubernetes
66
type: application
7-
version: 0.1.0
7+
version: 0.1.1
88
appVersion: 2.11.4
99
kubeVersion: ">=1.9.0"
1010
keywords:

‎charts/saleor-core/config/backup.sh

+180
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
#!/usr/bin/env bash
2+
# shellcheck disable=SC2086,SC2153
3+
4+
set -eo pipefail
5+
6+
export AWS_ACCESS_KEY_ID="${RESTIC_S3_ACCESS_KEY_ID}"
7+
export AWS_SECRET_ACCESS_KEY="${RESTIC_S3_SECRET_ACCESS_KEY}"
8+
export AWS_DEFAULT_REGION="${RESTIC_S3_REGION}"
9+
10+
function try_restic_initialization() {
11+
local restic_global_args=${1}
12+
local restic_host=${2}
13+
14+
if [[ -z "${restic_host}" ]]; then
15+
echo "Variable restic_host is required, exiting"
16+
exit 1
17+
fi
18+
19+
restic snapshots ${restic_global_args} --host ${restic_host} ||
20+
(
21+
echo "Initializing the restic repository" &&
22+
restic init ${restic_global_args}
23+
) &&
24+
(
25+
echo "The restic repository has already been initialized"
26+
)
27+
}
28+
29+
function init() {
30+
if [[ -z "${RESTIC_REPOSITORY}" ]]; then
31+
echo "Environment variable RESTIC_REPOSITORY not set, exiting"
32+
exit 1
33+
fi
34+
35+
if [[ -z "${RESTIC_S3_REGION}" ]]; then
36+
echo "Environment variable RESTIC_S3_REGION not set, exiting"
37+
exit 1
38+
fi
39+
40+
if [[ -z "${RESTIC_PASSWORD}" ]]; then
41+
echo "Environment variable RESTIC_PASSWORD not set, exiting"
42+
exit 1
43+
fi
44+
45+
if [[ -z "${RESTIC_S3_ACCESS_KEY_ID}" ]]; then
46+
echo "Environment variable RESTIC_S3_ACCESS_KEY_ID not set, exiting"
47+
exit 1
48+
fi
49+
50+
if [[ -z "${RESTIC_S3_SECRET_ACCESS_KEY}" ]]; then
51+
echo "Environment variable RESTIC_S3_SECRET_ACCESS_KEY not set, exiting"
52+
exit 1
53+
fi
54+
55+
if [[ -z "${RESTIC_HOST}" ]]; then
56+
echo "Environment variable RESTIC_HOST not set, exiting"
57+
exit 1
58+
fi
59+
60+
if [[ -z "${RESTIC_GLOBAL_ARGS}" ]]; then
61+
try_restic_initialization --host "${RESTIC_HOST}"
62+
else
63+
try_restic_initialization "${RESTIC_GLOBAL_ARGS}" --host "${RESTIC_HOST}"
64+
fi
65+
}
66+
67+
function execute_pg_dump() {
68+
local postgresql_password=${1}
69+
local postgresql_host=${2}
70+
local postgresql_port=${3}
71+
local postgresql_user=${4}
72+
local postgresql_database=${5}
73+
local postgresql_additional_args=${6}
74+
75+
if [[ ! -d /home/saleor/backups/database ]]; then
76+
mkdir -p /home/saleor/backups/database
77+
fi
78+
79+
echo "Dumping postgresql database with the following arguments:"
80+
echo "host: ${postgresql_host}"
81+
echo "port: ${postgresql_port}"
82+
echo "username: ${postgresql_user}"
83+
echo "dbname: ${postgresql_database}"
84+
85+
if [[ -z "${postgresql_additional_args}" ]]; then
86+
PGPASSWORD="${postgresql_password}" \
87+
pg_dump \
88+
--verbose \
89+
--host="${postgresql_host}" \
90+
--port="${postgresql_port}" \
91+
--username="${postgresql_user}" \
92+
--dbname="${postgresql_database}" \
93+
--no-password >/home/saleor/backups/database/saleor-core-postgresql.dump
94+
else
95+
echo "additional arguments: ${postgresql_additional_args}"
96+
PGPASSWORD="${postgresql_password}" \
97+
pg_dump \
98+
${postgresql_additional_args} \
99+
--host="${postgresql_host}" \
100+
--port="${postgresql_port}" \
101+
--username="${postgresql_user}" \
102+
--dbname="${postgresql_database}" \
103+
--no-password >/home/saleor/backups/database/saleor-core-postgresql.dump
104+
fi &&
105+
echo "Finished pg_dump successfully" ||
106+
echo "Finished pg_dump unsuccessfuly"
107+
}
108+
109+
function restic_save_db() {
110+
local restic_global_args=${1}
111+
local restic_host=${2}
112+
113+
restic backup \
114+
${restic_global_args} \
115+
--host "${restic_host}-db" \
116+
/home/saleor/backups/database/saleor-core-postgresql.dump &&
117+
echo "Saved media backup via restic tool successfully" ||
118+
echo "Saved media backup via restic tool unsuccessfully"
119+
}
120+
121+
function restic_save_media() {
122+
local restic_global_args=${1}
123+
local restic_host=${2}
124+
125+
restic backup \
126+
${restic_global_args} \
127+
--host "${restic_host}-media" \
128+
/app/media &&
129+
echo "Saved media backup via restic tool successfully" ||
130+
echo "Saved media backup via restic tool unsuccessfully"
131+
}
132+
133+
function display_snapshots() {
134+
restic snapshots --verbose --no-cache
135+
}
136+
137+
function main() {
138+
local do_postgresql_backup=${1}
139+
local postgresql_password=${2}
140+
local postgresql_host=${3}
141+
local postgresql_port=${4}
142+
local postgresql_user=${5}
143+
local postgresql_database=${6}
144+
local postgresql_additional_args=${7}
145+
local do_media_backup=${8}
146+
local restic_global_args=${9}
147+
local restic_host=${10}
148+
149+
init
150+
151+
if [[ "${do_postgresql_backup}" == "true" ]]; then
152+
echo "Postgresql backup is to be executed"
153+
154+
execute_pg_dump \
155+
"${postgresql_password}" \
156+
"${postgresql_host}" \
157+
"${postgresql_port}" \
158+
"${postgresql_user}" \
159+
"${postgresql_database}" \
160+
"${postgresql_additional_args}"
161+
fi
162+
if [[ "${do_media_backup}" == "true" ]]; then
163+
echo "Saving media files using restic"
164+
165+
restic_save_media \
166+
"${restic_global_args}" \
167+
"${restic_host}"
168+
fi
169+
if [[ "${do_postgresql_backup}" == "true" ]]; then
170+
echo "Saving postgresql backup using restic"
171+
172+
restic_save_db \
173+
"${restic_global_args}" \
174+
"${restic_host}"
175+
fi
176+
177+
display_snapshots
178+
}
179+
180+
main "${@}"

‎charts/saleor-core/templates/_helpers.tpl

+42
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,12 @@ env:
107107
{{- if .Values.existingSecret }}
108108
- name: SECRET_KEY
109109
value: "$(SALEOR_SECRET_KEY)"
110+
- name: RESTIC_PASSWORD
111+
value: ""
112+
- name: RESTIC_S3_ACCESS_KEY_ID
113+
value: ""
114+
- name: RESTIC_S3_SECRET_ACCESS_KEY
115+
value: ""
110116
{{- end }}
111117
{{- if and .Values.jobs.init.plugins.enabled .Values.externalServices.vatLayer.enabled }}
112118
- name: VATLAYER_API_KEY
@@ -158,6 +164,42 @@ env:
158164
{{- end }}
159165

160166

167+
{{/*
168+
Generate backup configuration
169+
*/}}
170+
{{- define "saleor-core.env.backup" -}}
171+
{{- if or .Values.backup.database.enabled .Values.backup.media.enabled }}
172+
- name: RESTIC_PASSWORD
173+
valueFrom:
174+
secretKeyRef:
175+
{{- if not .Values.existingSecret }}
176+
name: {{ include "saleor-core.fullname" . }}
177+
{{- else }}
178+
name: {{ .Values.existingSecret }}
179+
{{- end }}
180+
key: RESTIC_PASSWORD
181+
- name: RESTIC_S3_ACCESS_KEY_ID
182+
valueFrom:
183+
secretKeyRef:
184+
{{- if not .Values.existingSecret }}
185+
name: {{ include "saleor-core.fullname" . }}
186+
{{- else }}
187+
name: {{ .Values.existingSecret }}
188+
{{- end }}
189+
key: RESTIC_S3_ACCESS_KEY_ID
190+
- name: RESTIC_S3_SECRET_ACCESS_KEY
191+
valueFrom:
192+
secretKeyRef:
193+
{{- if not .Values.existingSecret }}
194+
name: {{ include "saleor-core.fullname" . }}
195+
{{- else }}
196+
name: {{ .Values.existingSecret }}
197+
{{- end }}
198+
key: RESTIC_S3_SECRET_ACCESS_KEY
199+
{{- end }}
200+
{{- end }}
201+
202+
161203
{{/*
162204
A script to check if the saleor-postgresql service is ready
163205
*/}}

‎charts/saleor-core/templates/backup/backup-cronjob.yaml

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
{{- if and .Values.backup.cron.enabled (or .Values.backup.database.enabled .Values.backup.media.enabled) }}
2+
apiVersion: batch/v1beta1
3+
kind: CronJob
4+
metadata:
5+
name: {{ include "saleor-core.fullname" . }}-backup-cronjob
6+
spec:
7+
schedule: {{ .Values.backup.cron.schedule | quote }}
8+
jobTemplate:
9+
spec:
10+
template:
11+
spec:
12+
{{- with .Values.imagePullSecrets }}
13+
imagePullSecrets:
14+
{{- toYaml . | nindent 8 }}
15+
{{- end }}
16+
serviceAccountName: {{ include "saleor-core.serviceAccountName" . }}
17+
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
18+
securityContext:
19+
{{- toYaml .Values.backup.podSecurityContext | nindent 12 }}
20+
volumes:
21+
# Volume for the images and unstructured data
22+
- name: saleor-data-media
23+
{{- if and .Values.persistence.enabled .Values.persistence.existingPvc }}
24+
persistentVolumeClaim:
25+
claimName: {{ .Values.persistence.existingPvc }}
26+
{{- else if .Values.persistence.enabled }}
27+
persistentVolumeClaim:
28+
claimName: {{ default (include "saleor-core.fullname" .) }}
29+
{{- else }}
30+
emptyDir:
31+
medium: Memory
32+
{{- end }}
33+
# ConfigMap for the backup script
34+
- name: backup
35+
configMap:
36+
name: {{ include "saleor-core.fullname" . }}-backup
37+
initContainers:
38+
# Wait for successful response from postgresql
39+
- name: "{{ include "saleor-core.fullname" . }}-backup-cronjob-init"
40+
securityContext:
41+
{{- toYaml .Values.backup.containerSecurityContext | nindent 12 }}
42+
{{- include "saleor-core.env" . | indent 10 }}
43+
image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag | default (cat "dev-" .Chart.AppVersion) | nospace }}"
44+
imagePullPolicy: {{ .Values.image.pullPolicy }}
45+
command:
46+
- /bin/bash
47+
- -c
48+
- >
49+
{{ include "saleor.postgresql.isReady" . | nindent 14 }}
50+
containers:
51+
- name: "{{ include "saleor-core.fullname" . }}-backup-cronjob"
52+
volumeMounts:
53+
- name: saleor-data-media
54+
mountPath: /app/media
55+
- name: backup
56+
mountPath: /home/saleor/backup.sh
57+
subPath: backup.sh
58+
readOnly: true
59+
securityContext:
60+
{{- toYaml .Values.backup.containerSecurityContext | nindent 12 }}
61+
{{- include "saleor-core.env" . | indent 10 }}
62+
{{- include "saleor-core.env.backup" . | indent 10 }}
63+
image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag | default (cat "dev-" .Chart.AppVersion) | nospace }}"
64+
imagePullPolicy: {{ .Values.image.pullPolicy }}
65+
command:
66+
- /bin/bash
67+
- /home/saleor/backup.sh
68+
args:
69+
- 'true'
70+
- '$(POSTGRESQL_PASSWORD)'
71+
- '$(POSTGRESQL_HOST)'
72+
- '$(POSTGRESQL_PORT)'
73+
- '$(POSTGRESQL_USER)'
74+
- '$(POSTGRESQL_DATABASE)'
75+
- '$(POSTGRESQL_ADDITIONAL_ARGS)'
76+
- 'true'
77+
- '$(RESTIC_GLOBAL_ARGS)'
78+
- '$(RESTIC_HOST)'
79+
{{- end }}

0 commit comments

Comments
 (0)