diff --git a/docker/k8s/vtctlclient/Dockerfile b/docker/k8s/vtctlclient/Dockerfile index 0315e572483..1af7e5ae076 100644 --- a/docker/k8s/vtctlclient/Dockerfile +++ b/docker/k8s/vtctlclient/Dockerfile @@ -1,4 +1,10 @@ FROM vitess/base AS base FROM debian:stretch-slim COPY --from=base /vt/bin/vtctlclient /usr/bin/ +RUN apt-get update && \ + apt-get upgrade -qq && \ + apt-get install jq -qq --no-install-recommends && \ + apt-get autoremove && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* CMD ["/usr/bin/vtctlclient"] diff --git a/helm/vitess/templates/_helpers.tpl b/helm/vitess/templates/_helpers.tpl index 4d2233c7e08..6a03c9677c5 100644 --- a/helm/vitess/templates/_helpers.tpl +++ b/helm/vitess/templates/_helpers.tpl @@ -11,6 +11,17 @@ {{end -}} {{- end -}} +############################# +# Repeat a string N times, where N is the total number +# of replicas. Len must be used on the calling end to +# get an int +############################# +{{- define "tablet-count" -}} +{{- range . -}} +{{- repeat (int .vttablet.replicas) "x" -}} +{{- end -}} +{{- end -}} + ############################# # Format a list of flag maps into a command line. ############################# diff --git a/helm/vitess/templates/_orchestrator-conf.tpl b/helm/vitess/templates/_orchestrator-conf.tpl index 5fc03569522..1f2e101c592 100644 --- a/helm/vitess/templates/_orchestrator-conf.tpl +++ b/helm/vitess/templates/_orchestrator-conf.tpl @@ -5,6 +5,7 @@ # set tuple values to more recognizable variables {{- $orc := index . 0 -}} {{- $namespace := index . 1 -}} +{{- $enableHeartbeat := index . 2 -}} apiVersion: v1 kind: ConfigMap @@ -117,7 +118,11 @@ data: "ReduceReplicationAnalysisCount": true, "RejectHostnameResolvePattern": "", "RemoveTextFromHostnameDisplay": ".mydomain.com:3306", +{{ if $enableHeartbeat }} "ReplicationLagQuery": "SELECT unix_timestamp() - floor(ts/1000000000) FROM `_vt`.heartbeat ORDER BY ts DESC LIMIT 1;", +{{ else }} + "ReplicationLagQuery": "", +{{ end }} "ServeAgentsHttp": false, "SkipBinlogEventsContaining": [ ], diff --git a/helm/vitess/templates/_vttablet.tpl b/helm/vitess/templates/_vttablet.tpl index 0f8de94150a..37a8d48d134 100644 --- a/helm/vitess/templates/_vttablet.tpl +++ b/helm/vitess/templates/_vttablet.tpl @@ -47,6 +47,7 @@ spec: {{- $config := index . 7 -}} {{- $pmm := index . 8 -}} {{- $orc := index . 9 -}} +{{- $totalTabletCount := index . 10 -}} # sanitize inputs to create tablet name {{- $cellClean := include "clean-label" $cell.name -}} @@ -54,6 +55,7 @@ spec: {{- $shardClean := include "clean-label" $shard.name -}} {{- $uid := "$(cat /vtdataroot/tabletdata/tablet-uid)" }} {{- $setName := printf "%s-%s-%s-%s" $cellClean $keyspaceClean $shardClean $tablet.type | lower -}} +{{- $shardName := printf "%s-%s-%s" $cellClean $keyspaceClean $shardClean | lower -}} {{- with $tablet.vttablet -}} @@ -103,7 +105,7 @@ spec: containers: {{ include "cont-mysql" (tuple $topology $cell $keyspace $shard $tablet $defaultVttablet $uid) | indent 8 }} -{{ include "cont-vttablet" (tuple $topology $cell $keyspace $shard $tablet $defaultVttablet $vitessTag $uid $namespace $config $orc) | indent 8 }} +{{ include "cont-vttablet" (tuple $topology $cell $keyspace $shard $tablet $defaultVttablet $vitessTag $uid $namespace $config $orc $totalTabletCount) | indent 8 }} {{ include "cont-mysql-errorlog" . | indent 8 }} {{ include "cont-mysql-slowlog" . | indent 8 }} {{ if $pmm.enabled }}{{ include "cont-pmm-client" (tuple $pmm $namespace) | indent 8 }}{{ end }} @@ -140,6 +142,96 @@ spec: shard: {{ $shardClean | quote }} type: {{ $tablet.type | quote }} +{{ if eq $tablet.type "replica" }} +--- +################################### +# vttablet InitShardMaster Job +################################### +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $shardName }}-init-shard-master +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: OnFailure + containers: + - name: init-shard-master + image: "vitess/vtctlclient:{{$vitessTag}}" + + command: ["bash"] + args: + - "-c" + - | + set -ex + + VTCTLD_SVC=vtctld.{{ $namespace }}:15999 + SECONDS=0 + TIMEOUT_SECONDS=600 + + # poll every 5 seconds to see if vtctld is ready + until vtctlclient -server $VTCTLD_SVC ListAllTablets {{ $cellClean }} > /dev/null 2>&1; do + if (( $SECONDS > $TIMEOUT_SECONDS )); then + echo "timed out waiting for vtctlclient to be ready" + exit 1 + fi + sleep 5 + done + + until [ $TABLETS_READY ]; do + # get all the tablets in the current cell + cellTablets="$(vtctlclient -server $VTCTLD_SVC ListAllTablets {{ $cellClean }})" + + # filter to only the tablets in our current shard + shardTablets=$( echo "$cellTablets" | awk 'substr( $5,1,{{ len $shardName }} ) == "{{ $shardName }}" {print $0}') + + # check for a master tablet from the ListAllTablets call + masterTablet=$( echo "$shardTablets" | awk '$4 == "master" {print $1}') + if [ $masterTablet ]; then + echo "'$masterTablet' is already the master tablet, exiting without running InitShardMaster" + exit + fi + + # check for a master tablet from the GetShard call + master_alias=$(vtctlclient -server $VTCTLD_SVC GetShard {{ $keyspaceClean }}/{{ $shard.name }} | jq '.master_alias.uid') + if [ $master_alias != "null" ]; then + echo "'$master_alias' is already the master tablet, exiting without running InitShardMaster" + exit + fi + + # count the number of newlines for the given shard to get the tablet count + tabletCount=$( echo "$shardTablets" | wc | awk '{print $1}') + + # check to see if the tablet count equals the expected tablet count + if [ $tabletCount == {{ $totalTabletCount }} ]; then + TABLETS_READY=true + else + if (( $SECONDS > $TIMEOUT_SECONDS )); then + echo "timed out waiting for tablets to be ready" + exit 1 + fi + + # wait 5 seconds for vttablets to continue getting ready + sleep 5 + fi + + done + + # find the tablet id for the "-replica-0" stateful set for a given cell, keyspace and shard + tablet_id=$( echo "$shardTablets" | awk 'substr( $5,1,{{ add (len $shardName) 10 }} ) == "{{ $shardName }}-replica-0" {print $1}') + + # initialize the shard master + until vtctlclient -server $VTCTLD_SVC InitShardMaster -force {{ $keyspaceClean }}/{{ $shard.name }} $tablet_id; do + if (( $SECONDS > $TIMEOUT_SECONDS )); then + echo "timed out waiting for InitShardMaster to succeed" + exit 1 + fi + sleep 5 + done + +{{- end -}} + {{- end -}} {{- end -}} @@ -244,6 +336,7 @@ spec: {{- $namespace := index . 8 -}} {{- $config := index . 9 -}} {{- $orc := index . 10 -}} +{{- $totalTabletCount := index . 11 -}} {{- $cellClean := include "clean-label" $cell.name -}} {{- with $tablet.vttablet -}} @@ -321,10 +414,14 @@ spec: -db-config-filtered-uname "vt_filtered" -db-config-filtered-dbname "vt_{{$keyspace.name}}" -db-config-filtered-charset "utf8" - -enable_semi_sync -enable_replication_reporter -{{ if $orc.enabled }} +{{ if $defaultVttablet.enableSemisync }} + -enable_semi_sync +{{ end }} +{{ if $defaultVttablet.enableHeartbeat }} -heartbeat_enable +{{ end }} +{{ if $orc.enabled }} -orc_api_url "http://orchestrator.{{ $namespace }}/api" -orc_discover_interval "5m" {{ end }} @@ -475,4 +572,4 @@ affinity: app: "vitess" component: "vttablet" -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/vitess/templates/vitess.yaml b/helm/vitess/templates/vitess.yaml index 407c855d99d..521bd188100 100644 --- a/helm/vitess/templates/vitess.yaml +++ b/helm/vitess/templates/vitess.yaml @@ -15,7 +15,7 @@ {{ include "orchestrator" (tuple $.Values.orchestrator) }} --- # create orchestrator config map -{{ include "orchestrator-config" (tuple $.Values.orchestrator $.Release.Namespace) }} +{{ include "orchestrator-config" (tuple $.Values.orchestrator $.Release.Namespace $.Values.vttablet.enableHeartbeat) }} --- # create a Service per StatefulSet replica {{ range $i := until (int $.Values.orchestrator.replicas) }} @@ -53,9 +53,14 @@ # Tablets for keyspaces {{ range $keyspace := $cell.keyspaces }} {{ range $shard := $keyspace.shards }} + + {{ $totalTabletCount := len (include "tablet-count" $shard.tablets) }} + + # now range through the tablets again to set them up {{ range $tablet := $shard.tablets }} --- -{{ include "vttablet" (tuple $.Values.topology $cell $keyspace $shard $tablet $.Values.vttablet $.Release.Namespace $.Values.config $.Values.pmm $.Values.orchestrator) }} +{{ include "vttablet" (tuple $.Values.topology $cell $keyspace $shard $tablet $.Values.vttablet $.Release.Namespace $.Values.config $.Values.pmm $.Values.orchestrator $totalTabletCount) }} + {{ end }} # range $tablet {{ end }} # range $shard {{ end }} # range $keyspace diff --git a/helm/vitess/values.yaml b/helm/vitess/values.yaml index 276e77781c8..0f4fcd84e02 100644 --- a/helm/vitess/values.yaml +++ b/helm/vitess/values.yaml @@ -130,6 +130,13 @@ vttablet: mysqlImage: "percona:5.7.20" # mysqlImage: "mysql:5.7.20" # mysqlImage: "mariadb:10.3.4" + + enableHeartbeat: false + + # This requires at least 2 instances of "replica" tablet types, otherwise semi-sync + # will block forever. "rdonly" tablets do not ACK. + enableSemisync: false + resources: # common production values 2-4CPU/4-8Gi RAM limits: