diff --git a/helm/vitess/examples/minikube.yaml b/helm/vitess/examples/minikube.yaml new file mode 100644 index 00000000000..114a7654cdc --- /dev/null +++ b/helm/vitess/examples/minikube.yaml @@ -0,0 +1,73 @@ +topology: + cells: + - name: "zone1" + etcd: + replicas: 1 + vtctld: + replicas: 1 + vtgate: + replicas: 1 + mysqlProtocol: + enabled: true + authType: "none" + keyspaces: + - name: "messagedb" + schema: |- + CREATE TABLE messages ( + page BIGINT(20) UNSIGNED, + time_created_ns BIGINT(20) UNSIGNED, + message VARCHAR(10000), + PRIMARY KEY (page, time_created_ns) + ) ENGINE=InnoDB + vschema: |- + { + "sharded": true, + "vindexes": { + "hash": { + "type": "hash" + } + }, + "tables": { + "messages": { + "column_vindexes": [ + { + "column": "page", + "name": "hash" + } + ] + } + } + } + shards: + - name: "0" + tablets: + - type: "replica" + vttablet: + replicas: 2 + +etcd: + replicas: 1 + resources: + +vtctld: + serviceType: "NodePort" + resources: + +vtgate: + serviceType: "NodePort" + resources: + +vttablet: + resources: + mysqlResources: + +pmm: + enabled: false + client: + resources: + server: + resources: + +orchestrator: + enabled: false + resources: diff --git a/helm/vitess/templates/NOTES.txt b/helm/vitess/templates/NOTES.txt index 5d884bb6373..c2b52cf7538 100644 --- a/helm/vitess/templates/NOTES.txt +++ b/helm/vitess/templates/NOTES.txt @@ -1,5 +1,5 @@ {{- $cell := (index .Values.topology.cells 0).name -}} -{{- $proxyURL := printf "http://localhost:8001/api/v1/proxy/namespaces/%s" .Release.Namespace -}} +{{- $proxyURL := printf "http://localhost:8001/api/v1/namespaces/%s" .Release.Namespace -}} Release name: {{.Release.Name}} @@ -8,7 +8,7 @@ To access administrative web pages, start a proxy with: Then use the following URLs: - vtctld: {{$proxyURL}}/services/vtctld:web/app/ - vtgate: {{$proxyURL}}/services/vtgate-{{$cell}}:web/ + vtctld: {{$proxyURL}}/services/vtctld:web/proxy/app/ + vtgate: {{$proxyURL}}/services/vtgate-{{$cell}}:web/proxy/ {{ if $.Values.orchestrator.enabled }}orchestrator: {{$proxyURL}}/services/orchestrator:web/{{ end }} {{ if $.Values.pmm.enabled }} pmm: {{$proxyURL}}/services/pmm:web/{{ end }} diff --git a/helm/vitess/templates/_keyspace.tpl b/helm/vitess/templates/_keyspace.tpl new file mode 100644 index 00000000000..91c20951110 --- /dev/null +++ b/helm/vitess/templates/_keyspace.tpl @@ -0,0 +1,135 @@ +################################### +# keyspace initializations +################################### + +{{- define "keyspace" -}} +{{- $cell := index . 0 -}} +{{- $keyspace := index . 1 -}} +{{- $defaultVtctlclient := index . 2 -}} +{{- $namespace := index . 3 -}} + +# sanitize inputs for labels +{{- $keyspaceClean := include "clean-label" $keyspace.name -}} + +{{- with $cell.vtctld -}} + +# define image to use +{{- $vitessTag := .vitessTag | default $defaultVtctlclient.vitessTag -}} + +{{- if $keyspace.schema }} +--- +################################### +# ApplySchema Job +################################### +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $keyspaceClean }}-apply-schema +spec: + backoffLimit: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - name: apply-schema + image: "vitess/vtctlclient:{{$vitessTag}}" + volumeMounts: +{{ include "user-secret-volumeMounts" $defaultVtctlclient.secrets | indent 10 }} + + command: ["bash"] + args: + - "-c" + - | + set -ex + + VTCTLD_SVC=vtctld.{{ $namespace }}:15999 + SECONDS=0 + TIMEOUT_SECONDS=600 + VTCTL_EXTRA_FLAGS=({{ include "format-flags-inline" $defaultVtctlclient.extraFlags }}) + + # poll every 5 seconds to see if vtctld is ready + until vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC ListAllTablets {{ $cell.name }} > /dev/null 2>&1; do + if (( $SECONDS > $TIMEOUT_SECONDS )); then + echo "timed out waiting for vtctlclient to be ready" + exit 1 + fi + sleep 5 + done + + while true; do + if (( $SECONDS > $TIMEOUT_SECONDS )); then + echo "timed out waiting for master" + exit 1 + fi + + # wait for all shards to have a master + {{- range $shard := $keyspace.shards }} + master_alias=$(vtctlclient ${VTLCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC GetShard {{ $keyspace.name }}/{{ $shard.name }} | jq '.master_alias.uid') + if [ "$master_alias" == "null" -o "$master_alias" == "" ]; then + echo "no master for '{{ $keyspace.name }}/{{ $shard.name }}' yet, continuing to wait" + sleep 5 + continue + fi + {{- end }} + + break + done + + vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC ApplySchema -sql "$(cat < /dev/null 2>&1; do + if (( $SECONDS > $TIMEOUT_SECONDS )); then + echo "timed out waiting for keyspace {{ $keyspace.name }} to be ready" + exit 1 + fi + sleep 5 + done + + vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC ApplyVSchema -vschema "$(cat < /dev/null 2>&1; do + if (( $SECONDS > $TIMEOUT_SECONDS )); then + echo "timed out waiting for vtctlclient to be ready" + exit 1 + fi + sleep 5 + done + + until [ $TABLETS_READY ]; do + # get all the tablets in the current cell + cellTablets="$(vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC ListAllTablets {{ $cell.name }})" + + # filter to only the tablets in our current shard + shardTablets=$( echo "$cellTablets" | awk 'substr( $5,1,{{ len $shardName }} ) == "{{ $shardName }}" {print $0}') + + # check for a master tablet from the ListAllTablets call + masterTablet=$( echo "$shardTablets" | awk '$4 == "master" {print $1}') + if [ $masterTablet ]; then + echo "'$masterTablet' is already the master tablet, exiting without running InitShardMaster" + exit + fi + + # check for a master tablet from the GetShard call + master_alias=$(vtctlclient ${VTLCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC GetShard {{ $keyspace.name }}/{{ $shard.name }} | jq '.master_alias.uid') + if [ "$master_alias" != "null" -a "$master_alias" != "" ]; then + echo "'$master_alias' is already the master tablet, exiting without running InitShardMaster" + exit + fi + + # count the number of newlines for the given shard to get the tablet count + tabletCount=$( echo "$shardTablets" | wc | awk '{print $1}') + + # check to see if the tablet count equals the expected tablet count + if [ $tabletCount == {{ $totalTabletCount }} ]; then + TABLETS_READY=true + else + if (( $SECONDS > $TIMEOUT_SECONDS )); then + echo "timed out waiting for tablets to be ready" + exit 1 + fi + + # wait 5 seconds for vttablets to continue getting ready + sleep 5 + fi + + done + + # find the tablet id for the "-replica-0" stateful set for a given cell, keyspace and shard + tablet_id=$( echo "$shardTablets" | awk 'substr( $5,1,{{ add (len $shardName) 10 }} ) == "{{ $shardName }}-replica-0" {print $1}') + + # initialize the shard master + until vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC InitShardMaster -force {{ $keyspace.name }}/{{ $shard.name }} $tablet_id; do + if (( $SECONDS > $TIMEOUT_SECONDS )); then + echo "timed out waiting for InitShardMaster to succeed" + exit 1 + fi + sleep 5 + done + volumes: +{{ include "user-secret-volumes" (.secrets | default $defaultVtctlclient.secrets) | indent 8 }} + + +{{- end -}} +{{- end -}} diff --git a/helm/vitess/templates/_vtgate.tpl b/helm/vitess/templates/_vtgate.tpl index 6c4f784b601..a0f861d6654 100644 --- a/helm/vitess/templates/_vtgate.tpl +++ b/helm/vitess/templates/_vtgate.tpl @@ -66,8 +66,10 @@ spec: {{ include "vtgate-affinity" (tuple $cellClean $cell.region) | indent 6 }} {{ if $cell.mysqlProtocol.enabled }} +{{ if eq $cell.mysqlProtocol.authType "secret" }} initContainers: {{ include "init-mysql-creds" (tuple $vitessTag $cell) | indent 8 }} +{{ end }} {{ end }} containers: @@ -109,7 +111,12 @@ spec: -grpc_port=15991 {{ if $cell.mysqlProtocol.enabled }} -mysql_server_port=3306 +{{ if eq $cell.mysqlProtocol.authType "secret" }} + -mysql_auth_server_impl="static" -mysql_auth_server_static_file="/mysqlcreds/creds.json" +{{ else if eq $cell.mysqlProtocol.authType "none" }} + -mysql_auth_server_impl="none" +{{ end }} {{ end }} -service_map="grpc-vtgateservice" -cells_to_watch={{$cell.name | quote}} diff --git a/helm/vitess/templates/_vttablet.tpl b/helm/vitess/templates/_vttablet.tpl index b33f3b35f71..c05789058d9 100644 --- a/helm/vitess/templates/_vttablet.tpl +++ b/helm/vitess/templates/_vttablet.tpl @@ -48,9 +48,8 @@ spec: {{- $config := index . 8 -}} {{- $pmm := index . 9 -}} {{- $orc := index . 10 -}} -{{- $totalTabletCount := index . 11 -}} -# sanitize inputs to create tablet name +# sanitize inputs for labels {{- $cellClean := include "clean-label" $cell.name -}} {{- $keyspaceClean := include "clean-label" $keyspace.name -}} {{- $shardClean := include "clean-label" $shard.name -}} @@ -63,8 +62,9 @@ spec: # define images to use {{- $vitessTag := .vitessTag | default $defaultVttablet.vitessTag -}} {{- $image := .image | default $defaultVttablet.image -}} +{{- $mysqlImage := .mysqlImage | default $defaultVttablet.mysqlImage -}} {{- $mysqlImage := .mysqlImage | default $defaultVttablet.mysqlImage }} - +--- ################################### # vttablet StatefulSet ################################### @@ -106,7 +106,7 @@ spec: containers: {{ include "cont-mysql" (tuple $topology $cell $keyspace $shard $tablet $defaultVttablet $uid) | indent 8 }} -{{ include "cont-vttablet" (tuple $topology $cell $keyspace $shard $tablet $defaultVttablet $defaultVtctlclient $vitessTag $uid $namespace $config $orc $totalTabletCount) | indent 8 }} +{{ include "cont-vttablet" (tuple $topology $cell $keyspace $shard $tablet $defaultVttablet $defaultVtctlclient $vitessTag $uid $namespace $config $orc) | indent 8 }} {{ include "cont-logrotate" . | indent 8 }} {{ include "cont-mysql-generallog" . | indent 8 }} {{ include "cont-mysql-errorlog" . | indent 8 }} @@ -150,108 +150,6 @@ spec: # conditionally add cron job {{ include "vttablet-backup-cron" (tuple $cellClean $keyspaceClean $shardClean $shardName $keyspace $shard $vitessTag $config.backup $namespace $defaultVtctlclient) }} -{{ if eq $tablet.type "replica" }} ---- -################################### -# vttablet InitShardMaster Job -################################### -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ $shardName }}-init-shard-master - labels: - app: vitess - component: vttablet - cell: {{ $cellClean | quote }} - keyspace: {{ $keyspaceClean | quote }} - shard: {{ $shardClean | quote }} -spec: - backoffLimit: 10 - template: - spec: - restartPolicy: OnFailure - containers: - - name: init-shard-master - image: "vitess/vtctlclient:{{$vitessTag}}" - imagePullPolicy: Always - volumeMounts: -{{ include "user-secret-volumeMounts" $defaultVtctlclient.secrets | indent 10 }} - - command: ["bash"] - args: - - "-c" - - | - set -ex - - VTCTLD_SVC=vtctld.{{ $namespace }}:15999 - SECONDS=0 - TIMEOUT_SECONDS=600 - VTCTL_EXTRA_FLAGS=({{ include "format-flags-inline" $defaultVtctlclient.extraFlags }}) - - # poll every 5 seconds to see if vtctld is ready - until vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC ListAllTablets {{ $cellClean }} > /dev/null 2>&1; do - if (( $SECONDS > $TIMEOUT_SECONDS )); then - echo "timed out waiting for vtctlclient to be ready" - exit 1 - fi - sleep 5 - done - - until [ $TABLETS_READY ]; do - # get all the tablets in the current cell - cellTablets="$(vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC ListAllTablets {{ $cellClean }})" - - # filter to only the tablets in our current shard - shardTablets=$( echo "$cellTablets" | awk 'substr( $5,1,{{ len $shardName }} ) == "{{ $shardName }}" {print $0}') - - # check for a master tablet from the ListAllTablets call - masterTablet=$( echo "$shardTablets" | awk '$4 == "master" {print $1}') - if [ $masterTablet ]; then - echo "'$masterTablet' is already the master tablet, exiting without running InitShardMaster" - exit - fi - - # check for a master tablet from the GetShard call - master_alias=$(vtctlclient ${VTLCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC GetShard {{ $keyspace.name }}/{{ $shard.name }} | jq '.master_alias.uid') - if [ $master_alias != "null" ]; then - echo "'$master_alias' is already the master tablet, exiting without running InitShardMaster" - exit - fi - - # count the number of newlines for the given shard to get the tablet count - tabletCount=$( echo "$shardTablets" | wc | awk '{print $1}') - - # check to see if the tablet count equals the expected tablet count - if [ $tabletCount == {{ $totalTabletCount }} ]; then - TABLETS_READY=true - else - if (( $SECONDS > $TIMEOUT_SECONDS )); then - echo "timed out waiting for tablets to be ready" - exit 1 - fi - - # wait 5 seconds for vttablets to continue getting ready - sleep 5 - fi - - done - - # find the tablet id for the "-replica-0" stateful set for a given cell, keyspace and shard - tablet_id=$( echo "$shardTablets" | awk 'substr( $5,1,{{ add (len $shardName) 10 }} ) == "{{ $shardName }}-replica-0" {print $1}') - - # initialize the shard master - until vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC InitShardMaster -force {{ $keyspace.name }}/{{ $shard.name }} $tablet_id; do - if (( $SECONDS > $TIMEOUT_SECONDS )); then - echo "timed out waiting for InitShardMaster to succeed" - exit 1 - fi - sleep 5 - done - volumes: -{{ include "user-secret-volumes" (.secrets | default $defaultVtctlclient.secrets) | indent 8 }} - -{{- end -}} - {{- end -}} {{- end -}} @@ -364,7 +262,6 @@ spec: {{- $namespace := index . 9 -}} {{- $config := index . 10 -}} {{- $orc := index . 11 -}} -{{- $totalTabletCount := index . 12 -}} {{- $cellClean := include "clean-label" $cell.name -}} {{- with $tablet.vttablet }} diff --git a/helm/vitess/templates/vitess.yaml b/helm/vitess/templates/vitess.yaml index 89899398b1c..2e2253c8e0a 100644 --- a/helm/vitess/templates/vitess.yaml +++ b/helm/vitess/templates/vitess.yaml @@ -52,15 +52,19 @@ # Tablets for keyspaces {{ range $keyspace := $cell.keyspaces }} + + # Keyspace initializations + {{ include "keyspace" (tuple $cell $keyspace $.Values.vtctlclient $.Release.Namespace) }} + {{ range $shard := $keyspace.shards }} {{ $totalTabletCount := len (include "tablet-count" $shard.tablets) }} + # Shard initializations + {{ include "shard" (tuple $cell $keyspace $shard $.Values.vtctlclient $.Release.Namespace $totalTabletCount) }} - # now range through the tablets again to set them up + # Tablet initializations {{ range $tablet := $shard.tablets }} ---- -{{ include "vttablet" (tuple $.Values.topology $cell $keyspace $shard $tablet $.Values.vttablet $.Values.vtctlclient $.Release.Namespace $.Values.config $.Values.pmm $.Values.orchestrator $totalTabletCount) }} - + {{ include "vttablet" (tuple $.Values.topology $cell $keyspace $shard $tablet $.Values.vttablet $.Values.vtctlclient $.Release.Namespace $.Values.config $.Values.pmm $.Values.orchestrator) }} {{ end }} # range $tablet {{ end }} # range $shard {{ end }} # range $keyspace diff --git a/helm/vitess/values.yaml b/helm/vitess/values.yaml index 902220c709b..820f9aaa2a0 100644 --- a/helm/vitess/values.yaml +++ b/helm/vitess/values.yaml @@ -84,6 +84,8 @@ topology: # enable or disable mysql protocol support, with accompanying auth details mysqlProtocol: enabled: false + authType: "secret" + # authType can be: none or k8s_secret. For k8s_secret, perform the following changes: # username: myuser # this is the secret that will be mounted as the user password # kubectl create secret generic myuser_password --from-literal=password=abc123 @@ -134,6 +136,7 @@ vtgate: # Default values for vtctlclient resources defined in 'topology' vtctlclient: + vitessTag: "latest" extraFlags: {} secrets: [] # secrets are mounted under /vt/usersecrets/{secretname}