From abf0e517c1280ee105d3dc3ae92d9357ceb84195 Mon Sep 17 00:00:00 2001 From: Mark Solters Date: Wed, 31 Oct 2018 13:40:15 -0400 Subject: [PATCH 1/2] Let PMM client wait for mysql.sock There is a race condition where PMM can come up before the `mysql.sock` file is ready. PMM will fail to start and then permanently crash. Instead, wait for the sock file to be available and then proceed with PMM initialization. Also, latest fix to the `vitess/pmm-client` image means we no longer need the `|| true` fix for `pmm-admin add`. Signed-off-by: Mark Solters --- helm/vitess/templates/_pmm.tpl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/helm/vitess/templates/_pmm.tpl b/helm/vitess/templates/_pmm.tpl index 810171cb99a..69c63b16489 100644 --- a/helm/vitess/templates/_pmm.tpl +++ b/helm/vitess/templates/_pmm.tpl @@ -169,8 +169,11 @@ spec: pmm-admin config --server pmm.{{ $namespace }} --force # creates a systemd service - # TODO: remove "|| true" after https://jira.percona.com/projects/PMM/issues/PMM-1985 is resolved - pmm-admin add mysql:metrics --user root --socket /vtdataroot/tabletdata/mysql.sock --force || true + until [ -e /vtdataroot/tabletdata/mysql.sock ]; do + echo "Waiting for mysql.sock" + sleep 1 + done + pmm-admin add mysql:metrics --user root --socket /vtdataroot/tabletdata/mysql.sock --force # keep the container alive but still responsive to stop requests trap : TERM INT; sleep infinity & wait @@ -183,4 +186,4 @@ spec: - name: vtdataroot mountPath: /vtdataroot -{{- end -}} \ No newline at end of file +{{- end -}} From b0c06ba0ab9e2ee790790a42db3f9df2d2b13b64 Mon Sep 17 00:00:00 2001 From: Mark Solters Date: Wed, 31 Oct 2018 13:51:14 -0400 Subject: [PATCH 2/2] Update pmm-client with procps fix Installing `procps` in the `vitess/pmm-client` image resolves issue https://jira.percona.com/projects/PMM/issues/PMM-1985 Signed-off-by: Mark Solters --- docker/k8s/pmm-client/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/k8s/pmm-client/Dockerfile b/docker/k8s/pmm-client/Dockerfile index e5e4418b7e4..82d6f482efc 100644 --- a/docker/k8s/pmm-client/Dockerfile +++ b/docker/k8s/pmm-client/Dockerfile @@ -7,6 +7,7 @@ COPY --from=k8s /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificate RUN apt-get update && \ apt-get upgrade -qq && \ + apt-get install -y procps && \ apt-get install wget -qq --no-install-recommends && \ wget https://www.percona.com/redir/downloads/pmm-client/1.15.0/binary/debian/stretch/x86_64/pmm-client_1.15.0-1.stretch_amd64.deb && \ dpkg -i pmm-client_1.15.0-1.stretch_amd64.deb && \