From e050355abfd21da674142ec0e892d2cfb0988789 Mon Sep 17 00:00:00 2001
From: Derek Perkins <derek@derekperkins.com>
Date: Thu, 8 Nov 2018 18:56:03 -0700
Subject: [PATCH] helm: add per shard backup cron jobs

Signed-off-by: Derek Perkins <derek@derekperkins.com>
---
 helm/vitess/README.md               | 46 ++++++++++++++++-
 helm/vitess/templates/_jobs.tpl     | 77 +++++++++++++++++++++++++++++
 helm/vitess/templates/_vttablet.tpl |  3 ++
 helm/vitess/values.yaml             |  9 ++++
 4 files changed, 134 insertions(+), 1 deletion(-)
 create mode 100644 helm/vitess/templates/_jobs.tpl

diff --git a/helm/vitess/README.md b/helm/vitess/README.md
index 425ba3a8f8b..9e58d6b7fed 100644
--- a/helm/vitess/README.md
+++ b/helm/vitess/README.md
@@ -174,14 +174,58 @@ topology:
 
 ### Enable backup/restore using Google Cloud Storage
 
+Enabling backups creates a cron job per shard that defaults to executing once per day at midnight.
+This can be overridden on a per shard level so you can stagger when backups occur.
+
 ```
 topology:
   cells:
-    ...
+    - name: "zone1"
+      ...
+      keyspaces:
+        - name: "unsharded_dbname"
+          shards:
+            - name: "0"
+              backup:
+                cron:
+                  schedule: "0 1 * * *"
+                  suspend: false
+              tablets:
+                - type: "replica"
+                  vttablet:
+                    replicas: 2
+        - name: "sharded_db"
+          shards:
+            - name: "-80"
+              backup:
+                cron:
+                  schedule: "0 2 * * *"
+                  suspend: false
+              tablets:
+                - type: "replica"
+                  vttablet:
+                    replicas: 2
+            - name: "80-"
+              backup:
+                cron:
+                  schedule: "0 3 * * *"
+                  suspend: false
+              tablets:
+                - type: "replica"
+                  vttablet:
+                    replicas: 2
 
 config:
   backup:
     enabled: true
+
+    cron:
+      # the default schedule runs daily at midnight unless overridden by the individual shard
+      schedule: "0 0 * * *"
+
+      # if this is set to true, the cron jobs are created, but never execute
+      suspend: false
+
     backup_storage_implementation: gcs
 
     # Google Cloud Storage bucket to use for backups
diff --git a/helm/vitess/templates/_jobs.tpl b/helm/vitess/templates/_jobs.tpl
new file mode 100644
index 00000000000..ebbe576c612
--- /dev/null
+++ b/helm/vitess/templates/_jobs.tpl
@@ -0,0 +1,77 @@
+###################################
+# backup cron
+###################################
+{{ define "vttablet-backup-cron" -}}
+# set tuple values to more recognizable variables
+{{- $cellClean := index . 0 -}}
+{{- $keyspaceClean := index . 1 -}}
+{{- $shardClean := index . 2 -}}
+{{- $shardName := index . 3 -}}
+{{- $keyspace := index . 4 -}}
+{{- $shard := index . 5 -}}
+{{- $vitessTag := index . 6 -}}
+{{- $backup := index . 7 -}}
+{{- $namespace := index . 8 -}}
+{{- $defaultVtctlclient := index . 9 }}
+
+{{ if $backup.enabled }}
+# create cron job for current shard
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: {{ $shardName }}-backup
+  labels:
+    app: vitess
+    component: vttablet
+    cell: {{ $cellClean | quote }}
+    keyspace: {{ $keyspaceClean | quote }}
+    shard: {{ $shardClean | quote }}
+
+spec:
+  schedule: {{ $shard.backup.cron.schedule | default $backup.cron.schedule | quote }}
+  concurrencyPolicy: Forbid
+  suspend: {{ $shard.backup.cron.suspend | default $backup.cron.suspend }}
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 20
+
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: vitess
+            component: vttablet
+            cell: {{ $cellClean | quote }}
+            keyspace: {{ $keyspaceClean | quote }}
+            shard: {{ $shardClean | quote }}
+        # pod spec
+        spec:
+          restartPolicy: Never
+{{ include "pod-security" . | indent 10 }}
+
+          containers:
+          - name: backup
+            image: "vitess/vtctlclient:{{$vitessTag}}"
+            volumeMounts:
+{{ include "user-secret-volumeMounts" $defaultVtctlclient.secrets | indent 14 }}
+
+            command: ["bash"]
+            args:
+              - "-c"
+              - |
+                set -ex
+
+                VTCTLD_SVC=vtctld.{{ $namespace }}:15999
+                VTCTL_EXTRA_FLAGS=({{ include "format-flags-inline" $defaultVtctlclient.extraFlags }})
+
+                vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC BackupShard {{ $keyspace.name }}/{{ $shard.name }}
+
+            resources:
+              requests:
+                cpu: 10m
+                memory: 20Mi
+
+{{ end }}
+
+{{- end -}}
diff --git a/helm/vitess/templates/_vttablet.tpl b/helm/vitess/templates/_vttablet.tpl
index 87e4dc52cb9..c8193de6f2c 100644
--- a/helm/vitess/templates/_vttablet.tpl
+++ b/helm/vitess/templates/_vttablet.tpl
@@ -147,6 +147,9 @@ spec:
       shard: {{ $shardClean | quote }}
       type: {{ $tablet.type | quote }}
 
+# conditionally add cron job
+{{ include "vttablet-backup-cron" (tuple $cellClean $keyspaceClean $shardClean $shardName $keyspace $shard $vitessTag $config.backup $namespace $defaultVtctlclient) }}
+
 {{ if eq $tablet.type "replica" }}
 ---
 ###################################
diff --git a/helm/vitess/values.yaml b/helm/vitess/values.yaml
index 74385c47874..f56501e6329 100644
--- a/helm/vitess/values.yaml
+++ b/helm/vitess/values.yaml
@@ -22,6 +22,15 @@ config:
 
     enabled: false
 
+    # this creates 1 cron job per shard that will execute a backup using vtctlclient
+    # on this schedule. The job itself uses almost no resources.
+    cron:
+      # the default schedule runs daily at midnight unless overridden by the individual shard
+      schedule: "0 0 * * *"
+
+      # if this is set to true, the cron jobs are created, but never execute
+      suspend: false
+
     # choose a backup service - valid values are gcs/s3
     # TODO: add file and ceph support
     # backup_storage_implementation: gcs