diff --git a/docker/k8s/Dockerfile b/docker/k8s/Dockerfile new file mode 100644 index 00000000000..ce7cd358d10 --- /dev/null +++ b/docker/k8s/Dockerfile @@ -0,0 +1,58 @@ +FROM vitess/base AS base + +FROM debian:stretch-slim + +# Set up Vitess environment (just enough to run pre-built Go binaries) +ENV VTROOT /vt +ENV VTDATAROOT /vtdataroot +ENV VTTOP /vt/src/github.com/youtube/vitess + +# Prepare directory structure. +RUN mkdir -p /vt && \ + mkdir -p /vt/bin && \ + mkdir -p /vt/config && \ + mkdir -p /vt/web && \ + mkdir -p /vtdataroot/tabletdata + +# Copy CA certs for https calls +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +# Copy binaries +COPY --from=base /vt/bin/mysqlctld /vt/bin/ +COPY --from=base /vt/bin/vtctld /vt/bin/ +COPY --from=base /vt/bin/vtctl /vt/bin/ +COPY --from=base /vt/bin/vtgate /vt/bin/ +COPY --from=base /vt/bin/vttablet /vt/bin/ +COPY --from=base /vt/bin/vtworker /vt/bin/ + +# copy web admin files +COPY --from=base $VTTOP/web /vt/web/ + +# copy vitess config +COPY --from=base $VTTOP/config/init_db.sql /vt/config/ + +# mysql flavor files for db specific .cnf settings +COPY --from=base $VTTOP/config/mycnf/master_mysql56.cnf /vt/config/mycnf/ +COPY --from=base $VTTOP/config/mycnf/master_mariadb.cnf /vt/config/mycnf/ + +# settings for different types of instances +COPY --from=base $VTTOP/config/mycnf/default.cnf /vt/config/mycnf/ +COPY --from=base $VTTOP/config/mycnf/master.cnf /vt/config/mycnf/ +COPY --from=base $VTTOP/config/mycnf/replica.cnf /vt/config/mycnf/ +COPY --from=base $VTTOP/config/mycnf/rdonly.cnf /vt/config/mycnf/ +COPY --from=base $VTTOP/config/mycnf/backup.cnf /vt/config/mycnf/ + +# settings to support rbr +COPY --from=base $VTTOP/config/mycnf/rbr.cnf /vt/config/mycnf/ + +# add vitess user and add permissions +RUN groupadd -r --gid 999 vitess && useradd -r -g vitess --uid 999 vitess && \ + chown -R vitess:vitess /vt; + +# TODO: remove when https://github.com/youtube/vitess/issues/3553 is fixed +RUN apt-get update && \ + apt-get upgrade -qq && \ + apt-get install mysql-client -qq --no-install-recommends && \ + apt-get autoremove && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/helm/vitess/Chart.yaml b/helm/vitess/Chart.yaml index 7a3cfc5004d..368ea4d0232 100644 --- a/helm/vitess/Chart.yaml +++ b/helm/vitess/Chart.yaml @@ -1,10 +1,13 @@ apiVersion: v1 name: vitess -version: 0.2.0 +version: 0.9.0 description: Single-Chart Vitess Cluster keywords: - vitess - mysql + - maria + - mariadb + - percona - sql - database - shard diff --git a/helm/vitess/README.md b/helm/vitess/README.md index 264d6b85dfa..018506672e2 100644 --- a/helm/vitess/README.md +++ b/helm/vitess/README.md @@ -14,12 +14,22 @@ It currently includes all dependencies (e.g. etcd) and Vitess components **WARNING: This chart should be considered Alpha. Upgrading a release of this chart may or may not delete all your data.** +## Prerequisites + +* Install [etcd-operator](https://github.com/coreos/etcd-operator) in the + namespace where you plan to install this chart. + ## Installing the Chart ```console -helm/vitess$ helm install . +helm/vitess$ helm install . -f site-values.yaml ``` +See the [Configuration](#configuration) section below for what you need to put +in `site-values.yaml`. +You can install the chart without site values, but it will only launch a +skeleton cluster without any keyspaces (logical databases). + ## Cleaning up After deleting an installation of the chart, the PersistentVolumeClaims remain. @@ -31,5 +41,179 @@ kubectl delete pvc -l app=vitess ## Configuration -See the comments in `values.yaml` for descriptions of the parameters. +You will need to provide a `site-values.yaml` file to specify your actual +logical database topology (e.g. whether to shard). +Here are examples of various configurations. To see additional options, +look at the default `values.yaml` file, which is well commented. + +### Unsharded keyspace + +``` +topology: + cells: + - name: "zone1" + etcd: + replicas: 3 + vtctld: + replicas: 1 + vtgate: + replicas: 3 + mysqlProtocol: + enabled: false + keyspaces: + - name: "unsharded-dbname" + shards: + - name: "0" + tablets: + - type: "replica" + vttablet: + replicas: 2 +``` + +### Unsharded + sharded keyspaces + +``` +topology: + cells: + - name: "zone1" + ... + keyspaces: + - name: "unsharded-dbname" + shards: + - name: "0" + tablets: + - type: "replica" + vttablet: + replicas: 2 + - name: "sharded-db" + shards: + - name: "-80" + tablets: + - type: "replica" + vttablet: + replicas: 2 + - name: "80-" + tablets: + - type: "replica" + vttablet: + replicas: 2 +``` + +### Separate pools of replicas and rdonly tablets + +``` +topology: + cells: + - name: "zone1" + ... + keyspaces: + - name: "unsharded-dbname" + shards: + - name: "0" + tablets: + - type: "replica" + vttablet: + replicas: 2 + - type: "rdonly" + vttablet: + replicas: 2 +``` + +### Use a custom database image and a specific Vitess release + +``` +topology: + cells: + ... + +vttablet: + vitessTag: "2.1" + mysqlImage: "percona:5.7.20" + flavor: percona +``` + +### Enable MySQL protocol support +``` +topology: + cells: + - name: "zone1" + ... + # enable or disable mysql protocol support, with accompanying auth details + mysqlProtocol: + enabled: false + username: myuser + # this is the secret that will be mounted as the user password + # kubectl create secret generic myuser_password --from-literal=password=abc123 + passwordSecret: myuser-password + + keyspaces: + ... +``` + +### Enable backup/restore using Google Cloud Storage + +``` +topology: + cells: + ... + +config: + backup: + enabled: true + backup_storage_implementation: gcs + + # Google Cloud Storage bucket to use for backups + gcs_backup_storage_bucket: vitess-backups + + # root prefix for all backup-related object names + gcs_backup_storage_root: vtbackups +``` + +### Custom requests/limits + +``` +topology: + cells: + ... + +vttablet: + resources: + # common production values 2-4CPU/4-8Gi RAM + limits: + cpu: 2 + memory: 4Gi + mysqlResources: + # common production values 4CPU/8-16Gi RAM + limits: + cpu: 4 + memory: 8Gi + # PVC for mysql + dataVolumeClaimAnnotations: + dataVolumeClaimSpec: + # pd-ssd (Google Cloud) + # managed-premium (Azure) + # standard (AWS) - not sure what the default class is for ssd + storageClassName: "default" + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: "10Gi" +``` + +### Custom PVC for MySQL data + +``` +topology: + cells: + ... + +vttablet: + dataVolumeClaimSpec: + # Google Cloud SSD + storageClassName: "pd-ssd" + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: "100Gi" +``` diff --git a/helm/vitess/orchestrator.conf.json b/helm/vitess/orchestrator.conf.json deleted file mode 100644 index bcdfeca631a..00000000000 --- a/helm/vitess/orchestrator.conf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "Debug": true, - "EnableSyslog": false, - "ListenAddress": ":3000", - "AgentsServerPort": ":3001", - "MySQLTopologyUser": "orc_client_user", - "MySQLTopologyPassword": "orc_client_user_password", - "MySQLTopologyCredentialsConfigFile": "", - "MySQLTopologySSLPrivateKeyFile": "", - "MySQLTopologySSLCertFile": "", - "MySQLTopologySSLCAFile": "", - "MySQLTopologySSLSkipVerify": true, - "MySQLTopologyUseMutualTLS": false, - "MySQLTopologyMaxPoolConnections": 3, - "DatabaselessMode__experimental": false, - "MySQLOrchestratorHost": "127.0.0.1", - "MySQLOrchestratorPort": 3306, - "MySQLOrchestratorDatabase": "orchestrator", - "MySQLOrchestratorUser": "orc_server_user", - "MySQLOrchestratorPassword": "orc_server_user_password", - "MySQLOrchestratorCredentialsConfigFile": "", - "MySQLOrchestratorSSLPrivateKeyFile": "", - "MySQLOrchestratorSSLCertFile": "", - "MySQLOrchestratorSSLCAFile": "", - "MySQLOrchestratorSSLSkipVerify": true, - "MySQLOrchestratorUseMutualTLS": false, - "MySQLConnectTimeoutSeconds": 1, - "DefaultInstancePort": 3306, - "SkipOrchestratorDatabaseUpdate": false, - "SlaveLagQuery": "", - "SlaveStartPostWaitMilliseconds": 1000, - "DiscoverByShowSlaveHosts": true, - "InstancePollSeconds": 12, - "ReadLongRunningQueries": true, - "UnseenInstanceForgetHours": 240, - "SnapshotTopologiesIntervalHours": 0, - "DiscoveryPollSeconds": 5, - "InstanceBulkOperationsWaitTimeoutSeconds": 10, - "ActiveNodeExpireSeconds": 12, - "HostnameResolveMethod": "none", - "MySQLHostnameResolveMethod": "@@report_host", - "SkipBinlogServerUnresolveCheck": true, - "ExpiryHostnameResolvesMinutes": 0, - "RejectHostnameResolvePattern": "", - "ReasonableReplicationLagSeconds": 10, - "ProblemIgnoreHostnameFilters": [], - "VerifyReplicationFilters": false, - "MaintenanceOwner": "orchestrator", - "ReasonableMaintenanceReplicationLagSeconds": 20, - "MaintenanceExpireMinutes": 10, - "MaintenancePurgeDays": 365, - "CandidateInstanceExpireMinutes": 60, - "AuditLogFile": "/tmp/orchestrator-audit.log", - "AuditToSyslog": false, - "AuditPageSize": 20, - "AuditPurgeDays": 365, - "RemoveTextFromHostnameDisplay": ".mydomain.com:3306", - "ReadOnly": false, - "AuthenticationMethod": "", - "HTTPAuthUser": "", - "HTTPAuthPassword": "", - "AuthUserHeader": "", - "PowerAuthUsers": [ - "*" - ], - "ClusterNameToAlias": { - "127.0.0.1": "test suite" - }, - "DetectClusterAliasQuery": "SELECT value FROM _vt.local_metadata WHERE name='ClusterAlias'", - "DetectClusterDomainQuery": "", - "DetectInstanceAliasQuery": "SELECT value FROM _vt.local_metadata WHERE name='Alias'", - "DetectPromotionRuleQuery": "SELECT value FROM _vt.local_metadata WHERE name='PromotionRule'", - "DataCenterPattern": "[.]([^.]+)[.][^.]+[.]mydomain[.]com", - "PhysicalEnvironmentPattern": "[.]([^.]+[.][^.]+)[.]mydomain[.]com", - "PromotionIgnoreHostnameFilters": [], - "DetectSemiSyncEnforcedQuery": "SELECT @@global.rpl_semi_sync_master_wait_no_slave AND @@global.rpl_semi_sync_master_timeout > 1000000", - "ServeAgentsHttp": false, - "AgentsUseSSL": false, - "AgentsUseMutualTLS": false, - "AgentSSLSkipVerify": false, - "AgentSSLPrivateKeyFile": "", - "AgentSSLCertFile": "", - "AgentSSLCAFile": "", - "AgentSSLValidOUs": [], - "UseSSL": false, - "UseMutualTLS": false, - "SSLSkipVerify": false, - "SSLPrivateKeyFile": "", - "SSLCertFile": "", - "SSLCAFile": "", - "SSLValidOUs": [], - "StatusEndpoint": "/api/status", - "StatusSimpleHealth": true, - "StatusOUVerify": false, - "HttpTimeoutSeconds": 60, - "AgentPollMinutes": 60, - "AgentAutoDiscover": false, - "UnseenAgentForgetHours": 6, - "StaleSeedFailMinutes": 60, - "SeedAcceptableBytesDiff": 8192, - "PseudoGTIDPattern": "drop view if exists .*?`_pseudo_gtid_hint__", - "PseudoGTIDMonotonicHint": "asc:", - "DetectPseudoGTIDQuery": "", - "BinlogEventsChunkSize": 10000, - "BufferBinlogEvents": true, - "SkipBinlogEventsContaining": [], - "ReduceReplicationAnalysisCount": true, - "FailureDetectionPeriodBlockMinutes": 60, - "RecoveryPollSeconds": 10, - "RecoveryPeriodBlockMinutes": 1, - "RecoveryPeriodBlockSeconds": 60, - "RecoveryIgnoreHostnameFilters": [], - "RecoverMasterClusterFilters": [ - ".*" - ], - "RecoverIntermediateMasterClusterFilters": [ - "_intermediate_master_pattern_" - ], - "OnFailureDetectionProcesses": [ - "echo 'Detected {failureType} on {failureCluster}. Affected replicas: {countSlaves}' >> /tmp/recovery.log" - ], - "PreFailoverProcesses": [ - "echo 'Will recover from {failureType} on {failureCluster}' >> /tmp/recovery.log" - ], - "PostFailoverProcesses": [ - "echo '(for all types) Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log" - ], - "PostUnsuccessfulFailoverProcesses": [], - "PostMasterFailoverProcesses": [ - "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Promoted: {successorHost}:{successorPort}' >> /tmp/recovery.log", - "vtctlclient -server vtctld:15999 TabletExternallyReparented {successorAlias}" - ], - "PostIntermediateMasterFailoverProcesses": [ - "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log" - ], - "CoMasterRecoveryMustPromoteOtherCoMaster": true, - "DetachLostSlavesAfterMasterFailover": true, - "ApplyMySQLPromotionAfterMasterFailover": true, - "MasterFailoverLostInstancesDowntimeMinutes": 0, - "PostponeSlaveRecoveryOnLagMinutes": 0, - "OSCIgnoreHostnameFilters": [], - "GraphiteAddr": "", - "GraphitePath": "", - "GraphiteConvertHostnameDotsToUnderscores": true -} diff --git a/helm/vitess/templates/NOTES.txt b/helm/vitess/templates/NOTES.txt index a08bd9830f4..a59ef214f5d 100644 --- a/helm/vitess/templates/NOTES.txt +++ b/helm/vitess/templates/NOTES.txt @@ -1,4 +1,4 @@ -{{- $cell := (index .Values.topology.cells 1).name -}} +{{- $cell := (index .Values.topology.cells 0).name -}} {{- $proxyURL := printf "http://localhost:8001/api/v1/proxy/namespaces/%s" .Release.Namespace -}} Release name: {{.Release.Name}} @@ -8,8 +8,6 @@ To access administrative web pages, start a proxy with: Then use the following URLs: -Kubernetes dashboard: http://localhost:8001/ui - -vtctld: {{$proxyURL}}/services/vtctld:web/ +vtctld: {{$proxyURL}}/services/vtctld:web/app/ vtgate: {{$proxyURL}}/services/vtgate-{{$cell}}:web/ diff --git a/helm/vitess/templates/_etcd.tpl b/helm/vitess/templates/_etcd.tpl index 9abefbf15eb..2145e9c41b9 100644 --- a/helm/vitess/templates/_etcd.tpl +++ b/helm/vitess/templates/_etcd.tpl @@ -1,149 +1,34 @@ +################################### +# etcd cluster managed by pre-installed etcd operator +################################### {{- define "etcd" -}} -{{- $ := index . 0 -}} -{{- $cell := index . 1 -}} -{{- with (index . 2) -}} -{{- $0 := $.Values.etcd -}} -{{- $replicas := .replicas | default $0.replicas -}} -# etcd -# Regular service for load balancing client connections. -kind: Service -apiVersion: v1 +# set tuple values to more recognizable variables +{{- $name := index . 0 -}} +{{- $replicas := index . 1 -}} +{{- $version := index . 2 -}} +{{- $resources := index . 3 -}} + +################################### +# EtcdCluster +################################### +apiVersion: "etcd.database.coreos.com/v1beta2" +kind: "EtcdCluster" metadata: - name: "etcd-{{$cell.name}}" - labels: - component: etcd - cell: {{$cell.name | quote}} - app: vitess + name: "etcd-{{ $name }}" spec: - ports: - - port: 4001 - selector: - component: etcd - cell: {{$cell.name | quote}} - app: vitess ---- -# Headless service for etcd cluster bootstrap. -kind: Service -apiVersion: v1 -metadata: - name: "etcd-{{$cell.name}}-srv" - labels: - component: etcd - cell: {{$cell.name | quote}} - app: vitess -spec: - clusterIP: None - ports: - - name: etcd-server - port: 7001 - selector: - component: etcd - cell: {{$cell.name | quote}} - app: vitess ---- -apiVersion: extensions/v1beta1 -kind: ReplicaSet -metadata: - name: "etcd-{{$cell.name}}" -spec: - replicas: {{$replicas}} - template: - metadata: - labels: - component: etcd - cell: {{$cell.name | quote}} - app: vitess - spec: - volumes: - - name: certs - hostPath: { path: {{$.Values.certsPath | quote}} } - containers: - - name: etcd - image: {{.image | default $0.image | quote}} - volumeMounts: - - name: certs - readOnly: true - # Mount root certs from the host OS into the location - # expected for our container OS (Debian): - mountPath: /etc/ssl/certs/ca-certificates.crt - resources: -{{ toYaml (.resources | default $0.resources) | indent 12 }} - command: - - bash - - "-c" - - | - set -ex - - ipaddr=$(hostname -i) - peer_url="http://$ipaddr:7001" - client_url="http://$ipaddr:4001" - - export ETCD_NAME=$HOSTNAME - export ETCD_DATA_DIR=/vt/vtdataroot/etcd-$ETCD_NAME - export ETCD_STRICT_RECONFIG_CHECK=true - export ETCD_ADVERTISE_CLIENT_URLS=$client_url - export ETCD_INITIAL_ADVERTISE_PEER_URLS=$peer_url - export ETCD_LISTEN_CLIENT_URLS=$client_url - export ETCD_LISTEN_PEER_URLS=$peer_url - - if [ -d $ETCD_DATA_DIR ]; then - # We've been restarted with an intact datadir. - # Just run without trying to do any bootstrapping. - echo "Resuming with existing data dir: $ETCD_DATA_DIR" - else - # This is the first run for this member. - - # If there's already a functioning cluster, join it. - echo "Checking for existing cluster by trying to join..." - if result=$(etcdctl -C http://etcd-{{$cell.name}}:4001 member add $ETCD_NAME $peer_url); then - [[ "$result" =~ ETCD_INITIAL_CLUSTER=\"([^\"]*)\" ]] && \ - export ETCD_INITIAL_CLUSTER="${BASH_REMATCH[1]}" - export ETCD_INITIAL_CLUSTER_STATE=existing - echo "Joining existing cluster: $ETCD_INITIAL_CLUSTER" - else - # Join failed. Assume we're trying to bootstrap. - - # First register with global topo, if we aren't global. - if [ "{{$cell.name}}" != "global" ]; then - echo "Registering cell "{{$cell.name}}" with global etcd..." - until etcdctl -C "http://etcd-global:4001" \ - set "/vt/cells/{{$cell.name}}" "http://etcd-{{$cell.name}}:4001"; do - echo "[$(date)] waiting for global etcd to register cell '{{$cell.name}}'" - sleep 1 - done - fi - - # Use DNS to bootstrap. - - # First wait for the desired number of replicas to show up. - echo "Waiting for {{$replicas}} replicas in SRV record for etcd-{{$cell.name}}-srv..." - until [ $(getsrv etcd-server tcp etcd-{{$cell.name}}-srv | wc -l) -eq {{$replicas}} ]; do - echo "[$(date)] waiting for {{$replicas}} entries in SRV record for etcd-{{$cell.name}}-srv" - sleep 1 - done - - export ETCD_DISCOVERY_SRV=etcd-{{$cell.name}}-srv - echo "Bootstrapping with DNS discovery:" - getsrv etcd-server tcp etcd-{{$cell.name}}-srv - fi - fi - - # We've set up the env as we want it. Now run. - exec etcd - lifecycle: - preStop: - exec: - command: - - bash - - "-c" - - | - # Find our member ID. - members=$(etcdctl -C http://etcd-{{$cell.name}}:4001 member list) - if [[ "$members" =~ ^([0-9a-f]+):\ name=$HOSTNAME ]]; then - member_id=${BASH_REMATCH[1]} - echo "Removing $HOSTNAME ($member_id) from etcd-{{$cell.name}} cluster..." - etcdctl -C http://etcd-{{$cell.name}}:4001 member remove $member_id - fi -{{- end -}} + size: {{ $replicas }} + version: {{ $version | quote }} + pod: + resources: +{{ toYaml ($resources) | indent 6 }} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + # prefer to stay away from other same-cell etcd pods + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + etcd_cluster: "etcd-{{ $name }}" {{- end -}} - diff --git a/helm/vitess/templates/_helpers.tpl b/helm/vitess/templates/_helpers.tpl index 49d7185a642..b6957755df7 100644 --- a/helm/vitess/templates/_helpers.tpl +++ b/helm/vitess/templates/_helpers.tpl @@ -1,22 +1,28 @@ # Helper templates +############################# # Format a flag map into a command line, # as expected by the golang 'flag' package. # Boolean flags must be given a value, such as "true" or "false". +############################# {{- define "format-flags" -}} {{- range $key, $value := . -}} -{{$key}}={{$value | quote}} {{end -}} {{- end -}} +############################# # Format a list of flag maps into a command line. +############################# {{- define "format-flags-all" -}} {{- range . }}{{template "format-flags" .}}{{end -}} {{- end -}} +############################# # Clean labels, making sure it starts and ends with [A-Za-z0-9]. # This is especially important for shard names, which can start or end with # '-' (like -80 or 80-), which would be an invalid kubernetes label. +############################# {{- define "clean-label" -}} {{- $replaced_label := . | replace "_" "-"}} {{- if hasPrefix "-" . -}} @@ -28,24 +34,228 @@ x{{$replaced_label}} {{- end -}} {{- end -}} -# Common init-container to set up vtdataroot volume. -{{- define "init-vtdataroot" -}} -{{- $image := . -}} -{ - "name": "init-vtdataroot", - "image": {{$image | quote}}, - "imagePullPolicy": "IfNotPresent", - "command": ["bash", "-c", " - set -ex; - mkdir -p $VTDATAROOT/tmp; - chown vitess:vitess $VTDATAROOT $VTDATAROOT/tmp; - "], - "volumeMounts": [ - { - "name": "vtdataroot", - "mountPath": "/vt/vtdataroot" - } - ] -} +############################# +# injects default vitess environment variables +############################# +{{- define "vitess-env" -}} +- name: VTROOT + value: "/vt" +- name: VTDATAROOT + value: "/vtdataroot" +- name: GOBIN + value: "/vt/bin" +- name: VT_MYSQL_ROOT + value: "/usr" +- name: PKG_CONFIG_PATH + value: "/vt/lib" {{- end -}} +############################# +# inject default pod security +############################# +{{- define "pod-security" -}} +securityContext: + runAsUser: 1000 + fsGroup: 2000 + runAsNonRoot: true +{{- end -}} + +############################# +# support region nodeAffinity if defined +############################# +{{- define "node-affinity" -}} +{{- $region := . -}} +{{ with $region }} +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "failure-domain.beta.kubernetes.io/region" + operator: In + values: [{{ $region | quote }}] +{{- end -}} +{{- end -}} + +############################# +# mycnf exec +############################# +{{- define "mycnf-exec" -}} + +if [ "$VT_DB_FLAVOR" = "percona" ]; then + FLAVOR_MYCNF=/vt/config/mycnf/master_mysql56.cnf + +elif [ "$VT_DB_FLAVOR" = "mysql" ]; then + FLAVOR_MYCNF=/vt/config/mycnf/master_mysql56.cnf + +elif [ "$VT_DB_FLAVOR" = "maria" ]; then + FLAVOR_MYCNF=/vt/config/mycnf/master_mariadb.cnf + +fi + +export EXTRA_MY_CNF="$FLAVOR_MYCNF:/vtdataroot/tabletdata/report-host.cnf:/vt/config/mycnf/rbr.cnf" + +{{- end -}} + +############################# +# +# all backup helpers below +# +############################# + +############################# +# backup flags - expects config.backup +############################# +{{- define "backup-flags" -}} + +{{ if .enabled }} +-restore_from_backup +-backup_storage_implementation=$VT_BACKUP_SERVICE + +{{ if eq .backup_storage_implementation "gcs" }} +-gcs_backup_storage_bucket=$VT_GCS_BACKUP_STORAGE_BUCKET +-gcs_backup_storage_root=$VT_GCS_BACKUP_STORAGE_ROOT + +{{ else if eq .backup_storage_implementation "s3" }} +-s3_backup_aws_region=$VT_S3_BACKUP_AWS_REGION +-s3_backup_storage_bucket=$VT_S3_BACKUP_STORAGE_BUCKET +-s3_backup_storage_root=$VT_S3_BACKUP_STORAGE_ROOT +-s3_backup_server_side_encryption=$VT_S3_BACKUP_SERVER_SIDE_ENCRYPTION +{{ end }} + +{{ end }} + +{{- end -}} + +############################# +# backup env - expects config.backup +############################# +{{- define "backup-env" -}} + +{{ if .enabled }} + +- name: VT_BACKUP_SERVICE + valueFrom: + configMapKeyRef: + name: vitess-cm + key: backup.backup_storage_implementation + +{{ if eq .backup_storage_implementation "gcs" }} + +- name: VT_GCS_BACKUP_STORAGE_BUCKET + valueFrom: + configMapKeyRef: + name: vitess-cm + key: backup.gcs_backup_storage_bucket +- name: VT_GCS_BACKUP_STORAGE_ROOT + valueFrom: + configMapKeyRef: + name: vitess-cm + key: backup.gcs_backup_storage_root + +{{ else if eq .backup_storage_implementation "s3" }} + +- name: VT_S3_BACKUP_AWS_REGION + valueFrom: + configMapKeyRef: + name: vitess-cm + key: backup.s3_backup_aws_region +- name: VT_S3_BACKUP_STORAGE_BUCKET + valueFrom: + configMapKeyRef: + name: vitess-cm + key: backup.s3_backup_storage_bucket +- name: VT_S3_BACKUP_STORAGE_ROOT + valueFrom: + configMapKeyRef: + name: vitess-cm + key: backup.s3_backup_storage_root +- name: VT_S3_BACKUP_SERVER_SIDE_ENCRYPTION + valueFrom: + configMapKeyRef: + name: vitess-cm + key: backup.s3_backup_server_side_encryption + +{{ end }} + +{{ end }} + +{{- end -}} + +############################# +# backup volume - expects config.backup +############################# +{{- define "backup-volume" -}} + +{{ if .enabled }} + + {{ if eq .backup_storage_implementation "gcs" }} + + {{ if .gcsSecret }} +- name: backup-creds + secret: + secretName: {{ .gcsSecret }} + {{ end }} + + {{ else if eq .backup_storage_implementation "s3" }} + + {{ if .s3Secret }} +- name: backup-creds + secret: + secretName: {{ .s3Secret }} + {{ end }} + + {{ end }} + +{{ end }} + +{{- end -}} + +############################# +# backup volumeMount - expects config.backup +############################# +{{- define "backup-volumeMount" -}} + +{{ if .enabled }} + + {{ if eq .backup_storage_implementation "gcs" }} + + {{ if .gcsSecret }} +- name: backup-creds + mountPath: /etc/secrets/creds + {{ end }} + + {{ else if eq .backup_storage_implementation "s3" }} + + {{ if .s3Secret }} +- name: backup-creds + mountPath: /etc/secrets/creds + {{ end }} + + {{ end }} + +{{ end }} + +{{- end -}} + +############################# +# backup exec +############################# +{{- define "backup-exec" -}} + +{{ if .enabled }} + +credsPath=/etc/secrets/creds/$(ls /etc/secrets/creds/ | head -1) + +{{ if eq .backup_storage_implementation "gcs" }} +export GOOGLE_APPLICATION_CREDENTIALS=$credsPath +cat $GOOGLE_APPLICATION_CREDENTIALS + +{{ else if eq .backup_storage_implementation "s3" }} +export AWS_SHARED_CREDENTIALS_FILE=$credsPath +cat $AWS_SHARED_CREDENTIALS_FILE + +{{ end }} + +{{ end }} + +{{- end -}} \ No newline at end of file diff --git a/helm/vitess/templates/_orchestrator.tpl b/helm/vitess/templates/_orchestrator.tpl deleted file mode 100644 index 64dd14054f4..00000000000 --- a/helm/vitess/templates/_orchestrator.tpl +++ /dev/null @@ -1,128 +0,0 @@ -{{- define "orchestrator" -}} -{{- $ := index . 0 -}} -{{- $cell := index . 1 -}} -{{- with index . 2 -}} -{{- $0 := $.Values.orchestrator -}} -{{- $dataVolumeType := .dataVolumeType | default $0.dataVolumeType -}} -# Orchestrator service -apiVersion: v1 -kind: Service -metadata: - name: orchestrator - labels: - component: orchestrator - app: vitess -spec: - ports: - - port: 80 - targetPort: 3000 - selector: - component: orchestrator - app: vitess -{{ if eq $dataVolumeType "PersistentVolume" }} ---- -# Orchestrator persistent volume claim -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: orchestrator-data - annotations: -{{ toYaml (.dataVolumeClaimAnnotations | default $0.dataVolumeClaimAnnotations) | indent 4 }} -spec: -{{ toYaml (.dataVolumeClaimSpec | default $0.dataVolumeClaimSpec) | indent 2 }} -{{ end }} ---- -# Orchestrator replication controller -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: orchestrator -spec: - replicas: {{.replicas | default $0.replicas}} - template: - metadata: - labels: - component: orchestrator - app: vitess - annotations: - pod.beta.kubernetes.io/init-containers: '[ - { - "name": "init-mysql", - "image": {{.image | default $0.image | quote}}, - "imagePullPolicy": "IfNotPresent", - "command": ["bash", "-c", " - set -ex\n - rm -rf /mnt/data/lost+found\n - if [[ ! -d /mnt/data/mysql ]]; then\n - cp -R /var/lib/mysql/* /mnt/data/\n - fi\n - chown -R mysql:mysql /mnt/data\n - "], - "volumeMounts": [ - {"name": "data", "mountPath": "/mnt/data"} - ] - } - ]' - spec: - containers: - - name: orchestrator - image: {{.image | default $0.image | quote}} - command: - - bash - - "-c" - - | - set -x - until mysqladmin -h 127.0.0.1 ping; do sleep 1; done - exec orchestrator http - livenessProbe: - httpGet: - path: "/" - port: 3000 - initialDelaySeconds: 300 - timeoutSeconds: 30 - readinessProbe: - httpGet: - path: "/" - port: 3000 - timeoutSeconds: 10 - volumeMounts: - - mountPath: /orc/conf - name: config - resources: -{{ toYaml (.resources | default $0.resources) | indent 12 }} - - name: mysql - image: {{.image | default $0.image | quote}} - volumeMounts: - - mountPath: /var/lib/mysql - name: data - livenessProbe: - exec: - command: ["mysqladmin", "ping"] - initialDelaySeconds: 60 - timeoutSeconds: 10 - resources: -{{ toYaml (.mysqlResources | default $0.mysqlResources) | indent 12 }} - command: ["mysqld"] - volumes: - - name: config - configMap: - name: orchestrator - - name: data -{{ if eq $dataVolumeType "PersistentVolume" }} - persistentVolumeClaim: - claimName: orchestrator-data -{{ else }} - emptyDir: {} -{{ end }} ---- -# Orchestrator ConfigMap -apiVersion: v1 -kind: ConfigMap -metadata: - name: orchestrator -data: - orchestrator.conf.json: | -{{ $.Files.Get "orchestrator.conf.json" | indent 4 }} -{{- end -}} -{{- end -}} - diff --git a/helm/vitess/templates/_vtctld.tpl b/helm/vitess/templates/_vtctld.tpl index 96584ede1d8..05dc0eb09e7 100644 --- a/helm/vitess/templates/_vtctld.tpl +++ b/helm/vitess/templates/_vtctld.tpl @@ -1,9 +1,23 @@ +################################### +# vtctld Service + Deployment +################################### {{- define "vtctld" -}} -{{- $ := index . 0 -}} +# set tuple values to more recognizable variables +{{- $topology := index . 0 -}} {{- $cell := index . 1 -}} -{{- with index . 2 -}} -{{- $0 := $.Values.vtctld -}} -# vtctld +{{- $defaultVtctld := index . 2 -}} +{{- $namespace := index . 3 -}} +{{- $config := index . 4 -}} + +{{- with $cell.vtctld -}} + +# define image to use +{{- $vitessTag := .vitessTag | default $defaultVtctld.vitessTag -}} +{{- $cellClean := include "clean-label" $cell.name -}} + +################################### +# vtctld Service +################################### kind: Service apiVersion: v1 metadata: @@ -20,112 +34,89 @@ spec: selector: component: vtctld app: vitess - type: {{.serviceType | default $0.serviceType}} + type: {{.serviceType | default $defaultVtctld.serviceType}} --- -apiVersion: extensions/v1beta1 +################################### +# vtctld Service + Deployment +################################### +apiVersion: apps/v1beta1 kind: Deployment metadata: name: vtctld spec: - replicas: {{.replicas | default $0.replicas}} + replicas: {{.replicas | default $defaultVtctld.replicas}} + selector: + matchLabels: + app: vitess + component: vtctld template: metadata: labels: - component: vtctld app: vitess - annotations: - pod.beta.kubernetes.io/init-containers: '[ -{{ include "init-vtdataroot" (.image | default $0.image) | indent 10 }}, - { - "name": "init-vtctld", - "image": {{.image | default $0.image | quote}}, - "imagePullPolicy": "IfNotPresent", - "command": ["bash", "-c", " - set -ex\n - rm -rf /vt/web/*\n - cp -R $VTTOP/web/* /vt/web/\n - cp /mnt/config/config.js /vt/web/vtctld/\n - "], - "volumeMounts": [ - { - "name": "config", - "mountPath": "/mnt/config" - }, - { - "name": "web", - "mountPath": "/vt/web" - } - ] - } - ]' + component: vtctld spec: +{{ include "pod-security" . | indent 6 }} +{{ include "vtctld-affinity" (tuple $cellClean $cell.region) | indent 6 }} containers: - name: vtctld - image: {{.image | default $0.image | quote}} + image: vitess/k8s:{{$vitessTag}} livenessProbe: httpGet: path: /debug/vars port: 15000 initialDelaySeconds: 30 timeoutSeconds: 5 + env: +{{ include "backup-env" $config.backup | indent 12 }} volumeMounts: - - name: syslog - mountPath: /dev/log - - name: vtdataroot - mountPath: /vt/vtdataroot - - name: web - mountPath: /vt/web - - name: certs - readOnly: true - # Mount root certs from the host OS into the location - # expected for our container OS (Debian): - mountPath: /etc/ssl/certs/ca-certificates.crt +{{ include "backup-volumeMount" $config.backup | indent 12 }} + resources: -{{ toYaml (.resources | default $0.resources) | indent 12 }} - securityContext: - runAsUser: 999 +{{ toYaml (.resources | default $defaultVtctld.resources) | indent 12 }} command: - bash - "-c" - | - set -ex + set -ex; + +{{ include "backup-exec" $config.backup | indent 14 }} + eval exec /vt/bin/vtctld $(cat < /mysqlcreds/creds.json + +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/helm/vitess/templates/_vttablet.tpl b/helm/vitess/templates/_vttablet.tpl index 620ca956e8e..11127576d11 100644 --- a/helm/vitess/templates/_vttablet.tpl +++ b/helm/vitess/templates/_vttablet.tpl @@ -1,251 +1,425 @@ -# vttablet pod spec (template for both StatefulSet and naked pod) -{{- define "vttablet-pod-spec" -}} -{{- $ := index . 0 -}} -{{- $cell := index . 1 -}} -{{- $keyspace := index . 2 -}} -{{- $shard := index . 3 -}} -{{- $tablet := index . 4 -}} -{{- $uid := index . 5 -}} -{{- with index . 6 -}} -{{- $0 := $.Values.vttablet -}} -{{- $controllerType := .controllerType | default $0.controllerType -}} -containers: - - name: vttablet - image: {{.image | default $0.image | quote}} - livenessProbe: - httpGet: - path: /debug/vars - port: 15002 - initialDelaySeconds: 60 - timeoutSeconds: 10 - volumeMounts: - - name: syslog - mountPath: /dev/log - - name: vtdataroot - mountPath: /vt/vtdataroot - - name: certs - readOnly: true - # Mount root certs from the host OS into the location - # expected for our container OS (Debian): - mountPath: /etc/ssl/certs/ca-certificates.crt - resources: -{{ toYaml (.resources | default $0.resources) | indent 6 }} - ports: - - name: web - containerPort: 15002 - - name: grpc - containerPort: 16002 - securityContext: - runAsUser: 999 - command: - - bash - - "-c" - - | - set -ex - eval exec /vt/bin/vttablet $(cat < $VTDATAROOT/init/tablet-uid\n - # Tell MySQL what hostname to report in SHOW SLAVE HOSTS.\n - # Orchestrator looks there, so it should match -tablet_hostname above.\n - echo report-host=$hostname.vttablet > $VTDATAROOT/init/report-host.cnf\n - "], - "volumeMounts": [ - { - "name": "vtdataroot", - "mountPath": "/vt/vtdataroot" - } - ] -} {{- end -}} -# vttablet StatefulSet -{{- define "vttablet-stateful-set" -}} -{{- $ := index . 0 -}} +################################### +# vttablet +################################### +{{- define "vttablet" -}} +# set tuple values to more recognizable variables +{{- $topology := index . 0 -}} {{- $cell := index . 1 -}} {{- $keyspace := index . 2 -}} {{- $shard := index . 3 -}} {{- $tablet := index . 4 -}} -{{- with $tablet.vttablet -}} -{{- $0 := $.Values.vttablet -}} +{{- $defaultVttablet := index . 5 -}} +{{- $namespace := index . 6 -}} +{{- $config := index . 7 -}} + +# sanitize inputs to create tablet name {{- $cellClean := include "clean-label" $cell.name -}} {{- $keyspaceClean := include "clean-label" $keyspace.name -}} {{- $shardClean := include "clean-label" $shard.name -}} +{{- $uid := "$(cat /vtdataroot/tabletdata/tablet-uid)" }} {{- $setName := printf "%s-%s-%s-%s" $cellClean $keyspaceClean $shardClean $tablet.type | lower -}} -{{- $uid := "$(cat $VTDATAROOT/init/tablet-uid)" }} + +{{- with $tablet.vttablet -}} + +# define images to use +{{- $vitessTag := .vitessTag | default $defaultVttablet.vitessTag -}} +{{- $image := .image | default $defaultVttablet.image -}} +{{- $mysqlImage := .mysqlImage | default $defaultVttablet.mysqlImage -}} + +################################### # vttablet StatefulSet +################################### apiVersion: apps/v1beta1 kind: StatefulSet metadata: - name: {{$setName | quote}} + name: {{ $setName | quote }} spec: serviceName: vttablet - replicas: {{.replicas | default $0.replicas}} + replicas: {{ .replicas | default $defaultVttablet.replicas }} + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: vitess + component: vttablet + cell: {{ $cellClean | quote }} + keyspace: {{ $keyspaceClean | quote }} + shard: {{ $shardClean | quote }} + type: {{ $tablet.type | quote }} template: metadata: labels: app: vitess component: vttablet - cell: {{$cellClean | quote}} - keyspace: {{$keyspace.name | quote}} - shard: {{$shardClean | quote}} - type: {{$tablet.type | quote}} - annotations: - pod.alpha.kubernetes.io/initialized: "true" - pod.beta.kubernetes.io/init-containers: '[ -{{ include "init-vtdataroot" (.image | default $0.image) | indent 10 }}, -{{ include "init-tablet-uid" (tuple (.image | default $0.image) $cell) | indent 10 }} - ]' + cell: {{ $cellClean | quote }} + keyspace: {{ $keyspaceClean | quote }} + shard: {{ $shardClean | quote }} + type: {{ $tablet.type | quote }} spec: -{{ include "vttablet-pod-spec" (tuple $ $cell $keyspace $shard $tablet $uid .) | indent 6 }} -{{ if eq (.dataVolumeType | default $0.dataVolumeType) "PersistentVolume" }} +{{ include "pod-security" . | indent 6 }} +{{ include "vttablet-affinity" (tuple $cellClean $keyspaceClean $shardClean $cell.region) | indent 6 }} + + initContainers: +{{ include "init-mysql" (tuple $vitessTag $cellClean $namespace) | indent 8 }} +{{ include "init-tablet-uid" (tuple $vitessTag $cell) | indent 8 }} + + containers: +{{ include "cont-mysql" (tuple $topology $cell $keyspace $shard $tablet $defaultVttablet $uid) | indent 8 }} +{{ include "cont-vttablet" (tuple $topology $cell $keyspace $shard $tablet $defaultVttablet $vitessTag $uid $namespace $config) | indent 8 }} +{{ include "cont-mysql-errorlog" . | indent 8 }} +{{ include "cont-mysql-slowlog" . | indent 8 }} + + volumes: + - name: vt + emptyDir: {} +{{ include "backup-volume" $config.backup | indent 8 }} + volumeClaimTemplates: - metadata: name: vtdataroot annotations: -{{ toYaml (.dataVolumeClaimAnnotations | default $0.dataVolumeClaimAnnotations) | indent 10 }} +{{ toYaml (.dataVolumeClaimAnnotations | default $defaultVttablet.dataVolumeClaimAnnotations) | indent 10 }} spec: -{{ toYaml (.dataVolumeClaimSpec | default $0.dataVolumeClaimSpec) | indent 8 }} -{{ end }} +{{ toYaml (.dataVolumeClaimSpec | default $defaultVttablet.dataVolumeClaimSpec) | indent 8 }} + +{{- end -}} +{{- end -}} + +################################### +# init-container to copy binaries for mysql +################################### +{{- define "init-mysql" -}} +{{- $vitessTag := index . 0 -}} +{{- $cellClean := index . 1 -}} +{{- $namespace := index . 2 -}} + +- name: "init-mysql" + image: "vitess/k8s:{{$vitessTag}}" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: vtdataroot + mountPath: "/vtdataroot" + - name: vt + mountPath: "/vttmp" + + command: ["bash"] + args: + - "-c" + - | + set -ex + # set up the directories vitess needs + mkdir -p /vttmp/bin + mkdir -p /vtdataroot/tabletdata + + # copy necessary assets to the volumeMounts + cp /vt/bin/mysqlctld /vttmp/bin/ + cp -R /vt/config /vttmp/ + + # make sure that etcd is initialized + eval exec /vt/bin/vtctl $(cat < /vtdataroot/tabletdata/tablet-uid + # Tell MySQL what hostname to report in SHOW SLAVE HOSTS. + echo report-host=$hostname.vttablet > /vtdataroot/tabletdata/report-host.cnf + # Orchestrator looks there, so it should match -tablet_hostname above. + +{{- end -}} + +########################## +# main vttablet container +########################## +{{- define "cont-vttablet" -}} + +{{- $topology := index . 0 -}} +{{- $cell := index . 1 -}} +{{- $keyspace := index . 2 -}} +{{- $shard := index . 3 -}} +{{- $tablet := index . 4 -}} +{{- $defaultVttablet := index . 5 -}} +{{- $vitessTag := index . 6 -}} +{{- $uid := index . 7 -}} +{{- $namespace := index . 8 -}} +{{- $config := index . 9 -}} + +{{- $cellClean := include "clean-label" $cell.name -}} +{{- with $tablet.vttablet -}} + +- name: vttablet + image: "vitess/k8s:{{$vitessTag}}" + livenessProbe: + httpGet: + path: /debug/vars + port: 15002 + initialDelaySeconds: 60 + timeoutSeconds: 10 + volumeMounts: + - name: vtdataroot + mountPath: "/vtdataroot" +{{ include "backup-volumeMount" $config.backup | indent 4 }} + + resources: +{{ toYaml (.resources | default $defaultVttablet.resources) | indent 6 }} + ports: + - name: web + containerPort: 15002 + - name: grpc + containerPort: 16002 + env: +{{ include "vitess-env" . | indent 4 }} +{{ include "backup-env" $config.backup | indent 4 }} + + - name: VT_DB_FLAVOR + valueFrom: + configMapKeyRef: + name: vitess-cm + key: db.flavor + + command: ["bash"] + args: + - "-c" + - | + set -ex + +{{ include "mycnf-exec" . | indent 6 }} +{{ include "backup-exec" $config.backup | indent 6 }} + + eval exec /vt/bin/vttablet $(cat <