helm: reparent on vttablet preStop hook

docker/k8s/Dockerfile

@@ -2,6 +2,14 @@ FROM vitess/base AS base
 
 FROM debian:stretch-slim
 
+# TODO: remove when https://github.com/vitessio/vitess/issues/3553 is fixed
+RUN apt-get update && \
+   apt-get upgrade -qq && \
+   apt-get install mysql-client -qq --no-install-recommends && \
+   apt-get autoremove && \
+   apt-get clean && \
+   rm -rf /var/lib/apt/lists/*
+
 # Set up Vitess environment (just enough to run pre-built Go binaries)
 ENV VTROOT /vt
 ENV VTDATAROOT /vtdataroot
@@ -50,11 +58,3 @@ COPY --from=base $VTTOP/config/mycnf/rbr.cnf /vt/config/mycnf/
 # add vitess user and add permissions
 RUN groupadd -r --gid 999 vitess && useradd -r -g vitess --uid 999 vitess && \
    chown -R vitess:vitess /vt;
-
-# TODO: remove when https://github.com/vitessio/vitess/issues/3553 is fixed
-RUN apt-get update && \
-    apt-get upgrade -qq && \
-    apt-get install mysql-client -qq --no-install-recommends && \
-    apt-get autoremove && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*

docker/k8s/mysqlctld/Dockerfile

@@ -2,6 +2,13 @@ FROM vitess/k8s AS k8s
 
 FROM debian:stretch-slim
 
+RUN apt-get update && \
+   apt-get upgrade -qq && \
+   apt-get install busybox -qq --no-install-recommends && \
+   apt-get autoremove && \
+   apt-get clean && \
+   rm -rf /var/lib/apt/lists/*
+
 # Set up Vitess environment (just enough to run pre-built Go binaries)
 ENV VTROOT /vt
 ENV VTDATAROOT /vtdataroot

docker/k8s/orchestrator/Dockerfile

@@ -2,22 +2,22 @@ FROM vitess/k8s AS k8s
 
 FROM debian:stretch-slim
 
+RUN apt-get update && \
+   apt-get upgrade -qq && \
+   apt-get install wget -qq --no-install-recommends && \
+   wget https://github.com/github/orchestrator/releases/download/v3.0.13/orchestrator_3.0.13_amd64.deb && \
+   dpkg -i orchestrator_3.0.13_amd64.deb && \
+   rm orchestrator_3.0.13_amd64.deb && \
+   apt-get purge wget -qq && \
+   apt-get autoremove -qq && \
+   apt-get clean && \
+   rm -rf /var/lib/apt/lists/*
+
 # Copy certs to allow https calls
 COPY --from=k8s /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 
 # Copy vtctlclient to be used to notify 
 COPY --from=k8s /vt/bin/vtctlclient /usr/bin/
 
-RUN apt-get update && \
-    apt-get upgrade -qq && \
-    apt-get install wget -qq --no-install-recommends && \
-    wget https://github.com/github/orchestrator/releases/download/v3.0.13/orchestrator_3.0.13_amd64.deb && \
-    dpkg -i orchestrator_3.0.13_amd64.deb && \
-    rm orchestrator_3.0.13_amd64.deb && \
-    apt-get purge wget -qq && \
-    apt-get autoremove -qq && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
 WORKDIR /usr/local/orchestrator
 CMD ["./orchestrator", "--config=/conf/orchestrator.conf.json", "http"]

docker/k8s/pmm-client/Dockerfile

@@ -2,17 +2,17 @@ FROM vitess/k8s AS k8s
 
 FROM debian:stretch-slim
 
+RUN apt-get update && \
+   apt-get upgrade -qq && \
+   apt-get install -y procps && \
+   apt-get install wget -qq --no-install-recommends && \
+   wget https://www.percona.com/redir/downloads/pmm-client/1.15.0/binary/debian/stretch/x86_64/pmm-client_1.15.0-1.stretch_amd64.deb && \
+   dpkg -i pmm-client_1.15.0-1.stretch_amd64.deb && \
+   rm pmm-client_1.15.0-1.stretch_amd64.deb && \
+   apt-get purge wget -qq && \
+   apt-get autoremove -qq && \
+   apt-get clean && \
+   rm -rf /var/lib/apt/lists/*
+
 # Copy CA certs for https calls
 COPY --from=k8s /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-
-RUN apt-get update && \
-    apt-get upgrade -qq && \
-    apt-get install -y procps && \
-    apt-get install wget -qq --no-install-recommends && \
-    wget https://www.percona.com/redir/downloads/pmm-client/1.15.0/binary/debian/stretch/x86_64/pmm-client_1.15.0-1.stretch_amd64.deb && \
-    dpkg -i pmm-client_1.15.0-1.stretch_amd64.deb && \
-    rm pmm-client_1.15.0-1.stretch_amd64.deb && \
-    apt-get purge wget -qq && \
-    apt-get autoremove -qq && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*

docker/k8s/vtctlclient/Dockerfile

@@ -1,10 +1,14 @@
-FROM vitess/base AS base
+FROM vitess/k8s AS k8s
+
 FROM debian:stretch-slim
-COPY --from=base /vt/bin/vtctlclient /usr/bin/
+
 RUN apt-get update && \
-    apt-get upgrade -qq && \
-    apt-get install jq -qq --no-install-recommends && \
-    apt-get autoremove && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+   apt-get upgrade -qq && \
+   apt-get install jq -qq --no-install-recommends && \
+   apt-get autoremove && \
+   apt-get clean && \
+   rm -rf /var/lib/apt/lists/*
+
+COPY --from=k8s /vt/bin/vtctlclient /usr/bin/
+
 CMD ["/usr/bin/vtctlclient"]

docker/k8s/vttablet/Dockerfile

@@ -2,6 +2,14 @@ FROM vitess/k8s AS k8s
 
 FROM debian:stretch-slim
 
+# TODO: remove when https://github.com/vitessio/vitess/issues/3553 is fixed
+RUN apt-get update && \
+   apt-get upgrade -qq && \
+   apt-get install mysql-client jq -qq --no-install-recommends && \
+   apt-get autoremove && \
+   apt-get clean && \
+   rm -rf /var/lib/apt/lists/*
+
 # Set up Vitess environment (just enough to run pre-built Go binaries)
 ENV VTROOT /vt
 ENV VTDATAROOT /vtdataroot
@@ -11,14 +19,7 @@ RUN mkdir -p /vt/bin
 
 # Copy binaries
 COPY --from=k8s /vt/bin/vttablet /vt/bin/
+COPY --from=k8s /vt/bin/vtctlclient /vt/bin/
 
 # Copy certs to allow https calls
 COPY --from=k8s /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-
-# TODO: remove when https://github.com/vitessio/vitess/issues/3553 is fixed
-RUN apt-get update && \
-   apt-get upgrade -qq && \
-   apt-get install mysql-client -qq --no-install-recommends && \
-   apt-get autoremove && \
-   apt-get clean && \
-   rm -rf /var/lib/apt/lists/*

helm/vitess/templates/_vttablet.tpl

@@ -263,6 +263,7 @@ spec:
 
       # copy necessary assets to the volumeMounts
       cp /vt/bin/mysqlctld /vttmp/bin/
+      cp /bin/busybox /vttmp/bin/
       cp -R /vt/config /vttmp/
 
 {{- end -}}
@@ -379,6 +380,64 @@ spec:
           name: vitess-cm
           key: db.flavor
 
+  lifecycle:
+    preStop:
+      exec:
+        command:
+          - "bash"
+          - "-c"
+          - |
+            set -x
+
+            VTCTLD_SVC=vtctld.{{ $namespace }}:15999
+
+            master_alias_json=$(/vt/bin/vtctlclient -server $VTCTLD_SVC GetShard {{ $keyspace.name }}/{{ $shard.name }})
+            master_cell=$(jq -r '.master_alias.cell' <<< "$master_alias_json")
+            master_uid=$(jq -r '.master_alias.uid' <<< "$master_alias_json")
+            master_alias=$master_cell-$master_uid
+
+            current_uid=$(cat /vtdataroot/tabletdata/tablet-uid)
+            current_alias={{ $cell.name }}-$current_uid
+
+            if [ $master_alias != $current_alias ]; then
+                # since this isn't the master, there's no reason to reparent
+                exit
+            fi
+
+            # TODO: add more robust health checks to make sure that we don't initiate a reparent
+            # if there isn't a healthy enough replica to take over
+            # - seconds behind master
+            # - use GTID_SUBTRACT
+
+            RETRY_COUNT=0
+            MAX_RETRY_COUNT=5
+
+            # retry reparenting
+            until [ $DONE_REPARENTING ]; do
+
+              # reparent before shutting down
+              /vt/bin/vtctlclient -server $VTCTLD_SVC PlannedReparentShard -keyspace_shard={{ $keyspace.name }}/{{ $shard.name }} -avoid_master=$current_alias
+
+              # if PlannedReparentShard succeeded, then don't retry
+              if [ $? -eq 0 ]; then
+                DONE_REPARENTING=true
+
+              # if we've reached the max retry count, exit unsuccessfully
+              elif [ $RETRY_COUNT -eq $MAX_RETRY_COUNT ]; then
+                exit 1
+
+              # otherwise, increment the retry count and sleep for 10 seconds
+              else
+                let RETRY_COUNT=RETRY_COUNT+1
+                sleep 10
+              fi
+
+            done
+
+            # delete the current tablet from topology. Not strictly necessary, but helps to prevent
+            # edge cases where there are two masters
+            /vt/bin/vtctlclient -server $VTCTLD_SVC DeleteTablet $current_alias
+
   command: ["bash"]
   args:
     - "-c"
@@ -462,6 +521,28 @@ spec:
           name: vitess-cm
           key: db.flavor
 
+  lifecycle:
+    preStop:
+      exec:
+        command:
+          - "bash"
+          - "-c"
+          - |
+            set -x
+
+            # block shutting down mysqlctld until vttablet shuts down first
+            until [ $VTTABLET_GONE ]; do
+
+              # poll every 5 seconds to see if vttablet is still running
+              /vt/bin/busybox wget --spider localhost:15002/debug/vars
+
+              if [ $? -ne 0 ]; then
+                VTTABLET_GONE=true
+              fi
+
+              sleep 5
+            done
+
   command: ["bash"]
   args:
     - "-c"