diff --git a/.github/workflows/legacy_local_example.yml b/.github/workflows/legacy_local_example.yml new file mode 100644 index 00000000000..e661453f43d --- /dev/null +++ b/.github/workflows/legacy_local_example.yml @@ -0,0 +1,51 @@ +name: legacy_local_example +on: [push, pull_request] +jobs: + + build: + name: Local Example on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.13 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + if [ ${{matrix.os}} = "ubuntu-latest" ]; then + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + elif [ ${{matrix.os}} = "macos-latest" ]; then + brew install mysql@5.7 make unzip etcd curl git wget + fi + go mod download + + - name: Run make minimaltools + run: | + make minimaltools + + - name: Build + run: | + make build + + - name: legacy_local_example + run: | + if [ ${{matrix.os}} = "macos-latest" ]; then + export PATH="/usr/local/opt/mysql@5.7/bin:$PATH" + fi + # Make sure that testing is entirely non-reliant on config + mv config config-moved + eatmydata -- test/legacy_local_example.sh diff --git a/examples/are-you-alive/go.mod b/examples/are-you-alive/go.mod index 5f3d04ddccd..9bb69e1307b 100644 --- a/examples/are-you-alive/go.mod +++ b/examples/are-you-alive/go.mod @@ -4,8 +4,6 @@ go 1.12 require ( github.com/go-sql-driver/mysql v1.5.0 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 ) diff --git a/examples/legacy_local/101_initial_cluster.sh b/examples/legacy_local/101_initial_cluster.sh new file mode 100755 index 00000000000..e903ea58a6f --- /dev/null +++ b/examples/legacy_local/101_initial_cluster.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script brings up zookeeper and all the vitess components +# required for a single shard deployment. + +source ./env.sh + +# start topo server +if [ "${TOPO}" = "zk2" ]; then + CELL=zone1 ./scripts/zk-up.sh +else + CELL=zone1 ./scripts/etcd-up.sh +fi + +# start vtctld +CELL=zone1 ./scripts/vtctld-up.sh + +# start vttablets for keyspace commerce +for i in 100 101 102; do + CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh + CELL=zone1 KEYSPACE=commerce TABLET_UID=$i ./scripts/vttablet-up.sh +done + +# set one of the replicas to master +vtctlclient -server localhost:15999 InitShardMaster -force commerce/0 zone1-100 + +# create the schema +vtctlclient -server localhost:15999 ApplySchema -sql-file create_commerce_schema.sql commerce + +# create the vschema +vtctlclient -server localhost:15999 ApplyVSchema -vschema_file vschema_commerce_initial.json commerce + +# start vtgate +CELL=zone1 ./scripts/vtgate-up.sh diff --git a/examples/local/201_customer_keyspace.sh b/examples/legacy_local/201_customer_keyspace.sh similarity index 95% rename from examples/local/201_customer_keyspace.sh rename to examples/legacy_local/201_customer_keyspace.sh index f948175195c..9223a3b221e 100755 --- a/examples/local/201_customer_keyspace.sh +++ b/examples/legacy_local/201_customer_keyspace.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/local/202_customer_tablets.sh b/examples/legacy_local/202_customer_tablets.sh similarity index 97% rename from examples/local/202_customer_tablets.sh rename to examples/legacy_local/202_customer_tablets.sh index ec77226342f..88885139d69 100755 --- a/examples/local/202_customer_tablets.sh +++ b/examples/legacy_local/202_customer_tablets.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/local/203_vertical_split.sh b/examples/legacy_local/203_vertical_split.sh similarity index 96% rename from examples/local/203_vertical_split.sh rename to examples/legacy_local/203_vertical_split.sh index 372c4817a90..5d8e33e60ee 100755 --- a/examples/local/203_vertical_split.sh +++ b/examples/legacy_local/203_vertical_split.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/local/204_vertical_migrate_replicas.sh b/examples/legacy_local/204_vertical_migrate_replicas.sh similarity index 95% rename from examples/local/204_vertical_migrate_replicas.sh rename to examples/legacy_local/204_vertical_migrate_replicas.sh index 45ada0aaf63..f54c413d7c3 100755 --- a/examples/local/204_vertical_migrate_replicas.sh +++ b/examples/legacy_local/204_vertical_migrate_replicas.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/local/205_vertical_migrate_master.sh b/examples/legacy_local/205_vertical_migrate_master.sh similarity index 95% rename from examples/local/205_vertical_migrate_master.sh rename to examples/legacy_local/205_vertical_migrate_master.sh index 211d27eb6a7..fdfdf6d231b 100755 --- a/examples/local/205_vertical_migrate_master.sh +++ b/examples/legacy_local/205_vertical_migrate_master.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/local/206_clean_commerce.sh b/examples/legacy_local/206_clean_commerce.sh similarity index 96% rename from examples/local/206_clean_commerce.sh rename to examples/legacy_local/206_clean_commerce.sh index 7111a4a91fc..8bccbe99b97 100755 --- a/examples/local/206_clean_commerce.sh +++ b/examples/legacy_local/206_clean_commerce.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/legacy_local/301_customer_sharded.sh b/examples/legacy_local/301_customer_sharded.sh new file mode 100755 index 00000000000..ce72172f42e --- /dev/null +++ b/examples/legacy_local/301_customer_sharded.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script creates vitess sequences for the auto_increment fields +# and alters the fields to no longer be auto_increment in preparation +# for horizontal sharding +# it also changes the customer vschema from unsharded to sharded and +# sets up the necessary vindexes + +vtctlclient -server localhost:15999 ApplySchema -sql-file create_commerce_seq.sql commerce +vtctlclient -server localhost:15999 ApplyVSchema -vschema_file vschema_commerce_seq.json commerce +vtctlclient -server localhost:15999 ApplySchema -sql-file create_customer_sharded.sql customer +vtctlclient -server localhost:15999 ApplyVSchema -vschema_file vschema_customer_sharded.json customer diff --git a/examples/legacy_local/302_new_shards.sh b/examples/legacy_local/302_new_shards.sh new file mode 100755 index 00000000000..5d8e6768969 --- /dev/null +++ b/examples/legacy_local/302_new_shards.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script brings up new tablets for the two new shards that we will +# be creating in the customer keyspace and copies the schema + +source ./env.sh + +for i in 300 301 302; do + CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh + SHARD=-80 CELL=zone1 KEYSPACE=customer TABLET_UID=$i ./scripts/vttablet-up.sh +done + +for i in 400 401 402; do + CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh + SHARD=80- CELL=zone1 KEYSPACE=customer TABLET_UID=$i ./scripts/vttablet-up.sh +done + +vtctlclient -server localhost:15999 InitShardMaster -force customer/-80 zone1-300 +vtctlclient -server localhost:15999 InitShardMaster -force customer/80- zone1-400 +vtctlclient -server localhost:15999 CopySchemaShard customer/0 customer/-80 +vtctlclient -server localhost:15999 CopySchemaShard customer/0 customer/80- diff --git a/examples/local/303_horizontal_split.sh b/examples/legacy_local/303_horizontal_split.sh similarity index 96% rename from examples/local/303_horizontal_split.sh rename to examples/legacy_local/303_horizontal_split.sh index aaa019559d4..9c9ed8eda7d 100755 --- a/examples/local/303_horizontal_split.sh +++ b/examples/legacy_local/303_horizontal_split.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/local/304_migrate_replicas.sh b/examples/legacy_local/304_migrate_replicas.sh similarity index 95% rename from examples/local/304_migrate_replicas.sh rename to examples/legacy_local/304_migrate_replicas.sh index 3aed031ebab..bcb7cf21c0a 100755 --- a/examples/local/304_migrate_replicas.sh +++ b/examples/legacy_local/304_migrate_replicas.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/local/305_migrate_master.sh b/examples/legacy_local/305_migrate_master.sh similarity index 95% rename from examples/local/305_migrate_master.sh rename to examples/legacy_local/305_migrate_master.sh index a9da079ea7c..8592e1350a8 100755 --- a/examples/local/305_migrate_master.sh +++ b/examples/legacy_local/305_migrate_master.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/legacy_local/306_down_shard_0.sh b/examples/legacy_local/306_down_shard_0.sh new file mode 100755 index 00000000000..267de94bb90 --- /dev/null +++ b/examples/legacy_local/306_down_shard_0.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# this script brings down the tablets for customer/0 keyspace + +for i in 200 201 202; do + CELL=zone1 TABLET_UID=$i ./scripts/vttablet-down.sh + CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-down.sh +done + diff --git a/examples/legacy_local/307_delete_shard_0.sh b/examples/legacy_local/307_delete_shard_0.sh new file mode 100755 index 00000000000..36cb0a67863 --- /dev/null +++ b/examples/legacy_local/307_delete_shard_0.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script deletes the old shard 0 which has been replaced by 2 shards + +vtctlclient -server localhost:15999 DeleteShard -recursive customer/0 diff --git a/examples/legacy_local/401_teardown.sh b/examples/legacy_local/401_teardown.sh new file mode 100755 index 00000000000..629b9ba62f8 --- /dev/null +++ b/examples/legacy_local/401_teardown.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# We should not assume that any of the steps have been executed. +# This makes it possible for a user to cleanup at any point. + +source ./env.sh + +./scripts/vtgate-down.sh + +for tablet in 100 200 300 400; do + if vtctlclient -server localhost:15999 GetTablet zone1-$tablet >/dev/null 2>&1 ; then + # The zero tablet is up. Try to shutdown 0-2 tablet + mysqlctl + for i in 0 1 2; do + uid=$[$tablet + $i] + CELL=zone1 TABLET_UID=$uid ./scripts/vttablet-down.sh + CELL=zone1 TABLET_UID=$uid ./scripts/mysqlctl-down.sh + done + fi +done + +./scripts/vtctld-down.sh + +if [ "${TOPO}" = "zk2" ]; then + CELL=zone1 ./scripts/zk-down.sh +else + CELL=zone1 ./scripts/etcd-down.sh +fi + +# pedantic check: grep for any remaining processes + +if [ ! -z "$VTDATAROOT" ]; then + + if pgrep -f -l "$VTDATAROOT" > /dev/null; then + echo "ERROR: Stale processes detected! It is recommended to manuallly kill them:" + pgrep -f -l "$VTDATAROOT" + else + echo "All good! It looks like every process has shut down" + fi + + # shellcheck disable=SC2086 + rm -r ${VTDATAROOT:?}/* + +fi + +disown -a diff --git a/examples/legacy_local/README.md b/examples/legacy_local/README.md new file mode 100644 index 00000000000..245dbd4bb9e --- /dev/null +++ b/examples/legacy_local/README.md @@ -0,0 +1,10 @@ +# Local Vitess Cluster + +This directory contains example scripts to bring up a Vitess cluster on your +local machine, which may be useful for experimentation. These scripts can +also serve as a starting point for configuring Vitess into your preferred +deployment strategy or toolset. + +See the [Run Vitess Locally](https://vitess.io/docs/tutorials/local/) +tutorial ("Start a Vitess cluster" section) for instructions on using these scripts. + diff --git a/examples/legacy_local/ceph_backup_config.json b/examples/legacy_local/ceph_backup_config.json new file mode 100644 index 00000000000..d71c6170699 --- /dev/null +++ b/examples/legacy_local/ceph_backup_config.json @@ -0,0 +1,6 @@ +{ + "accessKey" : "AccessKey", + "secretKey" : "SecretKey", + "endPoint" : "URL", + "useSSL" : true +} diff --git a/examples/legacy_local/create_commerce_schema.sql b/examples/legacy_local/create_commerce_schema.sql new file mode 100644 index 00000000000..e62e7d2e8bf --- /dev/null +++ b/examples/legacy_local/create_commerce_schema.sql @@ -0,0 +1,18 @@ +create table product( + sku varbinary(128), + description varbinary(128), + price bigint, + primary key(sku) +) ENGINE=InnoDB; +create table customer( + customer_id bigint not null auto_increment, + email varbinary(128), + primary key(customer_id) +) ENGINE=InnoDB; +create table corder( + order_id bigint not null auto_increment, + customer_id bigint, + sku varbinary(128), + price bigint, + primary key(order_id) +) ENGINE=InnoDB; diff --git a/examples/legacy_local/create_commerce_seq.sql b/examples/legacy_local/create_commerce_seq.sql new file mode 100644 index 00000000000..b4e66c771b6 --- /dev/null +++ b/examples/legacy_local/create_commerce_seq.sql @@ -0,0 +1,4 @@ +create table customer_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence'; +insert into customer_seq(id, next_id, cache) values(0, 1000, 100); +create table order_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence'; +insert into order_seq(id, next_id, cache) values(0, 1000, 100); diff --git a/examples/legacy_local/create_customer_sharded.sql b/examples/legacy_local/create_customer_sharded.sql new file mode 100644 index 00000000000..9d3931c7c94 --- /dev/null +++ b/examples/legacy_local/create_customer_sharded.sql @@ -0,0 +1,2 @@ +alter table customer change customer_id customer_id bigint not null; +alter table corder change order_id order_id bigint not null; diff --git a/examples/legacy_local/create_test_table.sql b/examples/legacy_local/create_test_table.sql new file mode 100644 index 00000000000..bfb24b766ad --- /dev/null +++ b/examples/legacy_local/create_test_table.sql @@ -0,0 +1,7 @@ +CREATE TABLE messages ( + page BIGINT(20) UNSIGNED, + time_created_ns BIGINT(20) UNSIGNED, + message VARCHAR(10000), + PRIMARY KEY (page, time_created_ns) +) ENGINE=InnoDB + diff --git a/examples/legacy_local/drop_commerce_tables.sql b/examples/legacy_local/drop_commerce_tables.sql new file mode 100644 index 00000000000..3393ac8141a --- /dev/null +++ b/examples/legacy_local/drop_commerce_tables.sql @@ -0,0 +1,2 @@ +drop table customer; +drop table corder; diff --git a/examples/legacy_local/env.sh b/examples/legacy_local/env.sh new file mode 100644 index 00000000000..b825325539f --- /dev/null +++ b/examples/legacy_local/env.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +hostname=`hostname -f` +vtctld_web_port=15000 +export VTDATAROOT="${VTDATAROOT:-${PWD}/vtdataroot}" + +function fail() { + echo "ERROR: $1" + exit 1 +} + +if [[ $EUID -eq 0 ]]; then + fail "This script refuses to be run as root. Please switch to a regular user." +fi + +# mysqld might be in /usr/sbin which will not be in the default PATH +PATH="/usr/sbin:$PATH" +for binary in mysqld etcd etcdctl curl vtctlclient vttablet vtgate vtctld mysqlctl; do + command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/docs/get-started/local/ for install instructions." +done; + +if [ "${TOPO}" = "zk2" ]; then + # Each ZooKeeper server needs a list of all servers in the quorum. + # Since we're running them all locally, we need to give them unique ports. + # In a real deployment, these should be on different machines, and their + # respective hostnames should be given. + zkcfg=(\ + "1@$hostname:28881:38881:21811" \ + "2@$hostname:28882:38882:21812" \ + "3@$hostname:28883:38883:21813" \ + ) + printf -v zkcfg ",%s" "${zkcfg[@]}" + zkcfg=${zkcfg:1} + + zkids='1 2 3' + + # Set topology environment parameters. + ZK_SERVER="localhost:21811,localhost:21812,localhost:21813" + # shellcheck disable=SC2034 + TOPOLOGY_FLAGS="-topo_implementation zk2 -topo_global_server_address ${ZK_SERVER} -topo_global_root /vitess/global" + + mkdir -p $VTDATAROOT/tmp +else + ETCD_SERVER="localhost:2379" + TOPOLOGY_FLAGS="-topo_implementation etcd2 -topo_global_server_address $ETCD_SERVER -topo_global_root /vitess/global" + + mkdir -p "${VTDATAROOT}/tmp" + mkdir -p "${VTDATAROOT}/etcd" +fi diff --git a/examples/legacy_local/grpc_static_auth.json b/examples/legacy_local/grpc_static_auth.json new file mode 100644 index 00000000000..e6586acfb83 --- /dev/null +++ b/examples/legacy_local/grpc_static_auth.json @@ -0,0 +1,6 @@ +[ + { + "Username": "vitess", + "Password": "vitess_password" + } +] diff --git a/examples/legacy_local/grpc_static_client_auth.json b/examples/legacy_local/grpc_static_client_auth.json new file mode 100644 index 00000000000..b808a87607f --- /dev/null +++ b/examples/legacy_local/grpc_static_client_auth.json @@ -0,0 +1,4 @@ +{ + "Username": "vitess", + "Password": "vitess_password" +} diff --git a/examples/legacy_local/mysql_auth_server_static_creds.json b/examples/legacy_local/mysql_auth_server_static_creds.json new file mode 100644 index 00000000000..cded6bc22c5 --- /dev/null +++ b/examples/legacy_local/mysql_auth_server_static_creds.json @@ -0,0 +1,33 @@ +{ + "mysql_user": [ + { + "MysqlNativePassword": "*9E128DA0C64A6FCCCDCFBDD0FC0A2C967C6DB36F", + "Password": "mysql_password", + "UserData": "mysql_user" + } + ], + "mysql_user2": [ + { + "Password": "mysql_password", + "UserData": "mysql_user" + } + ], + "mysql_user3": [ + { + "MysqlNativePassword": "*9E128DA0C64A6FCCCDCFBDD0FC0A2C967C6DB36F", + "UserData": "mysql_user" + } + ], + + "vt_appdebug": [ + { + "Password": "vtappdebug_password", + "UserData": "vt_appdebug" + }, + { + "SourceHost": "localhost", + "Password": "", + "UserData": "vt_appdebug" + } + ] +} diff --git a/examples/legacy_local/scripts/etcd-down.sh b/examples/legacy_local/scripts/etcd-down.sh new file mode 100755 index 00000000000..e5cb4792f91 --- /dev/null +++ b/examples/legacy_local/scripts/etcd-down.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the etcd servers started by etcd-up.sh. + +source ./env.sh + +echo "Stopping etcd..." +kill -9 `cat $VTDATAROOT/tmp/etcd.pid` diff --git a/examples/legacy_local/scripts/etcd-up.sh b/examples/legacy_local/scripts/etcd-up.sh new file mode 100755 index 00000000000..72105a1ed86 --- /dev/null +++ b/examples/legacy_local/scripts/etcd-up.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a quorum of ZooKeeper servers. + +source ./env.sh + +cell=${CELL:-'test'} +export ETCDCTL_API=2 + +# Check that etcd is not already running +curl "http://${ETCD_SERVER}" > /dev/null 2>&1 && fail "etcd is already running. Exiting." + +etcd --enable-v2=true --data-dir "${VTDATAROOT}/etcd/" --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" > "${VTDATAROOT}"/tmp/etcd.out 2>&1 & +PID=$! +echo $PID > "${VTDATAROOT}/tmp/etcd.pid" +sleep 5 + +echo "add /vitess/global" +etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/global & + +echo "add /vitess/$cell" +etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/$cell & + +# And also add the CellInfo description for the cell. +# If the node already exists, it's fine, means we used existing data. +echo "add $cell CellInfo" +set +e +# shellcheck disable=SC2086 +vtctl $TOPOLOGY_FLAGS AddCellInfo \ + -root /vitess/$cell \ + -server_address "${ETCD_SERVER}" \ + $cell +set -e + +echo "etcd start done..." + + diff --git a/examples/legacy_local/scripts/mysqlctl-down.sh b/examples/legacy_local/scripts/mysqlctl-down.sh new file mode 100755 index 00000000000..0ff8e1b0cfb --- /dev/null +++ b/examples/legacy_local/scripts/mysqlctl-down.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the mysqld and vttablet instances +# created by vttablet-up.sh + +source ./env.sh + +mysqlctl -tablet_uid $TABLET_UID shutdown + diff --git a/examples/legacy_local/scripts/mysqlctl-up.sh b/examples/legacy_local/scripts/mysqlctl-up.sh new file mode 100755 index 00000000000..1bea78bc1f2 --- /dev/null +++ b/examples/legacy_local/scripts/mysqlctl-up.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a single shard vttablet deployment. + +source ./env.sh + +cell=${CELL:-'test'} +uid=$TABLET_UID +mysql_port=$[17000 + $uid] +printf -v alias '%s-%010d' $cell $uid +printf -v tablet_dir 'vt_%010d' $uid + +mkdir -p $VTDATAROOT/backups + +echo "Starting MySQL for tablet $alias..." +action="init" + +if [ -d $VTDATAROOT/$tablet_dir ]; then + echo "Resuming from existing vttablet dir:" + echo " $VTDATAROOT/$tablet_dir" + action='start' +fi + +mysqlctl \ + -log_dir $VTDATAROOT/tmp \ + -tablet_uid $uid \ + -mysql_port $mysql_port \ + $action diff --git a/examples/legacy_local/scripts/vtctld-down.sh b/examples/legacy_local/scripts/vtctld-down.sh new file mode 100755 index 00000000000..26d6a9905f8 --- /dev/null +++ b/examples/legacy_local/scripts/vtctld-down.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops vtctld. + +source ./env.sh + +echo "Stopping vtctld..." +kill -9 `cat $VTDATAROOT/tmp/vtctld.pid` diff --git a/examples/legacy_local/scripts/vtctld-up.sh b/examples/legacy_local/scripts/vtctld-up.sh new file mode 100755 index 00000000000..6ad693801c7 --- /dev/null +++ b/examples/legacy_local/scripts/vtctld-up.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that starts vtctld. + +source ./env.sh + +cell=${CELL:-'test'} +grpc_port=15999 + +echo "Starting vtctld..." +# shellcheck disable=SC2086 +vtctld \ + $TOPOLOGY_FLAGS \ + -cell $cell \ + -workflow_manager_init \ + -workflow_manager_use_election \ + -service_map 'grpc-vtctl' \ + -backup_storage_implementation file \ + -file_backup_storage_root $VTDATAROOT/backups \ + -log_dir $VTDATAROOT/tmp \ + -port $vtctld_web_port \ + -grpc_port $grpc_port \ + -pid_file $VTDATAROOT/tmp/vtctld.pid \ + > $VTDATAROOT/tmp/vtctld.out 2>&1 & diff --git a/examples/legacy_local/scripts/vtgate-down.sh b/examples/legacy_local/scripts/vtgate-down.sh new file mode 100755 index 00000000000..4508080b0d5 --- /dev/null +++ b/examples/legacy_local/scripts/vtgate-down.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the instance started by vtgate-up.sh. + +source ./env.sh + +# Stop vtgate. +echo "Stopping vtgate..." +kill `cat $VTDATAROOT/tmp/vtgate.pid` diff --git a/examples/legacy_local/scripts/vtgate-up.sh b/examples/legacy_local/scripts/vtgate-up.sh new file mode 100755 index 00000000000..07bc557b4fa --- /dev/null +++ b/examples/legacy_local/scripts/vtgate-up.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that starts a single vtgate. + +source ./env.sh + +cell=${CELL:-'test'} +web_port=15001 +grpc_port=15991 +mysql_server_port=15306 +mysql_server_socket_path="/tmp/mysql.sock" + +# Start vtgate. +# shellcheck disable=SC2086 +vtgate \ + $TOPOLOGY_FLAGS \ + -log_dir $VTDATAROOT/tmp \ + -log_queries_to_file $VTDATAROOT/tmp/vtgate_querylog.txt \ + -port $web_port \ + -grpc_port $grpc_port \ + -mysql_server_port $mysql_server_port \ + -mysql_server_socket_path $mysql_server_socket_path \ + -cell $cell \ + -cells_to_watch $cell \ + -tablet_types_to_wait MASTER,REPLICA \ + -gateway_implementation discoverygateway \ + -service_map 'grpc-vtgateservice' \ + -pid_file $VTDATAROOT/tmp/vtgate.pid \ + -mysql_auth_server_impl none \ + > $VTDATAROOT/tmp/vtgate.out 2>&1 & + +# Block waiting for vtgate to be listening +# Not the same as healthy + +echo "Waiting for vtgate to be up..." +while true; do + curl -I "http://$hostname:$web_port/debug/status" >/dev/null 2>&1 && break + sleep 0.1 +done; +echo "vtgate is up!" + +echo "Access vtgate at http://$hostname:$web_port/debug/status" + +disown -a diff --git a/examples/legacy_local/scripts/vttablet-down.sh b/examples/legacy_local/scripts/vttablet-down.sh new file mode 100755 index 00000000000..32bc844f4d3 --- /dev/null +++ b/examples/legacy_local/scripts/vttablet-down.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the mysqld and vttablet instances +# created by vttablet-up.sh + +source ./env.sh + +printf -v tablet_dir 'vt_%010d' $TABLET_UID +pid=`cat $VTDATAROOT/$tablet_dir/vttablet.pid` + +kill $pid + +# Wait for vttablet to die. +while ps -p $pid > /dev/null; do sleep 1; done + + diff --git a/examples/legacy_local/scripts/vttablet-up.sh b/examples/legacy_local/scripts/vttablet-up.sh new file mode 100755 index 00000000000..bbf623043df --- /dev/null +++ b/examples/legacy_local/scripts/vttablet-up.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./env.sh + +cell=${CELL:-'test'} +keyspace=${KEYSPACE:-'test_keyspace'} +shard=${SHARD:-'0'} +uid=$TABLET_UID +mysql_port=$[17000 + $uid] +port=$[15000 + $uid] +grpc_port=$[16000 + $uid] +printf -v alias '%s-%010d' $cell $uid +printf -v tablet_dir 'vt_%010d' $uid +tablet_hostname='' +printf -v tablet_logfile 'vttablet_%010d_querylog.txt' $uid + +tablet_type=replica +if [[ "${uid: -1}" -gt 1 ]]; then + tablet_type=rdonly +fi + +echo "Starting vttablet for $alias..." +# shellcheck disable=SC2086 +vttablet \ + $TOPOLOGY_FLAGS \ + -log_dir $VTDATAROOT/tmp \ + -log_queries_to_file $VTDATAROOT/tmp/$tablet_logfile \ + -tablet-path $alias \ + -tablet_hostname "$tablet_hostname" \ + -init_keyspace $keyspace \ + -init_shard $shard \ + -init_tablet_type $tablet_type \ + -health_check_interval 5s \ + -enable_semi_sync \ + -enable_replication_reporter \ + -backup_storage_implementation file \ + -file_backup_storage_root $VTDATAROOT/backups \ + -restore_from_backup \ + -port $port \ + -grpc_port $grpc_port \ + -service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \ + -pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \ + -vtctld_addr http://$hostname:$vtctld_web_port/ \ + > $VTDATAROOT/$tablet_dir/vttablet.out 2>&1 & + +# Block waiting for the tablet to be listening +# Not the same as healthy + +for i in $(seq 0 300); do + curl -I "http://$hostname:$port/debug/status" >/dev/null 2>&1 && break + sleep 0.1 +done + +# check one last time +curl -I "http://$hostname:$port/debug/status" || fail "tablet could not be started!" diff --git a/examples/legacy_local/scripts/zk-down.sh b/examples/legacy_local/scripts/zk-down.sh new file mode 100755 index 00000000000..dea665d48e7 --- /dev/null +++ b/examples/legacy_local/scripts/zk-down.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the ZooKeeper servers started by zk-up.sh. + +source ./env.sh + +# Stop ZooKeeper servers. +echo "Stopping zk servers..." +for zkid in $zkids; do + zkctl -zk.myid $zkid -zk.cfg $zkcfg -log_dir $VTDATAROOT/tmp shutdown +done + diff --git a/examples/legacy_local/scripts/zk-up.sh b/examples/legacy_local/scripts/zk-up.sh new file mode 100755 index 00000000000..8793dacdd31 --- /dev/null +++ b/examples/legacy_local/scripts/zk-up.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a quorum of ZooKeeper servers. + +source ./env.sh + +cell=${CELL:-'test'} + +# Start ZooKeeper servers. +# The "zkctl init" command won't return until the server is able to contact its +# peers, so we need to start them all in the background and then wait for them. +echo "Starting zk servers..." +for zkid in $zkids; do + action='init' + printf -v zkdir 'zk_%03d' $zkid + if [ -f $VTDATAROOT/$zkdir/myid ]; then + echo "Resuming from existing ZK data dir:" + echo " $VTDATAROOT/$zkdir" + action='start' + fi + zkctl -zk.myid $zkid -zk.cfg $zkcfg -log_dir $VTDATAROOT/tmp $action \ + > $VTDATAROOT/tmp/zkctl_$zkid.out 2>&1 & + pids[$zkid]=$! +done + +# Wait for all the zkctl commands to return. +echo "Waiting for zk servers to be ready..." + +for zkid in $zkids; do + if ! wait ${pids[$zkid]}; then + echo "ZK server number $zkid failed to start. See log:" + echo " $VTDATAROOT/tmp/zkctl_$zkid.out" + fi +done + +echo "Started zk servers." + +# Add the CellInfo description for the $CELL cell. +# If the node already exists, it's fine, means we used existing data. +set +e +# shellcheck disable=SC2086 +vtctl $TOPOLOGY_FLAGS AddCellInfo \ + -root /vitess/$cell \ + -server_address $ZK_SERVER \ + $cell +set -e + +echo "Configured zk servers." diff --git a/examples/legacy_local/topo-etcd2.sh b/examples/legacy_local/topo-etcd2.sh new file mode 100644 index 00000000000..4b6ebc62fc4 --- /dev/null +++ b/examples/legacy_local/topo-etcd2.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a single shard vttablet deployment. + +export TOPO='etcd2' + + diff --git a/examples/legacy_local/topo-zk2.sh b/examples/legacy_local/topo-zk2.sh new file mode 100644 index 00000000000..1a7cedfe2cd --- /dev/null +++ b/examples/legacy_local/topo-zk2.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a single shard vttablet deployment. + +export TOPO='zk2' + + diff --git a/examples/legacy_local/vschema.json b/examples/legacy_local/vschema.json new file mode 100644 index 00000000000..17e5dedf0c8 --- /dev/null +++ b/examples/legacy_local/vschema.json @@ -0,0 +1,18 @@ +{ + "sharded": true, + "vindexes": { + "hash": { + "type": "hash" + } + }, + "tables": { + "messages": { + "column_vindexes": [ + { + "column": "page", + "name": "hash" + } + ] + } + } +} diff --git a/examples/legacy_local/vschema_commerce_initial.json b/examples/legacy_local/vschema_commerce_initial.json new file mode 100644 index 00000000000..0d1bae44428 --- /dev/null +++ b/examples/legacy_local/vschema_commerce_initial.json @@ -0,0 +1,7 @@ +{ + "tables": { + "product": {}, + "customer": {}, + "corder": {} + } +} diff --git a/examples/legacy_local/vschema_commerce_seq.json b/examples/legacy_local/vschema_commerce_seq.json new file mode 100644 index 00000000000..2e564fa6c96 --- /dev/null +++ b/examples/legacy_local/vschema_commerce_seq.json @@ -0,0 +1,11 @@ +{ + "tables": { + "customer_seq": { + "type": "sequence" + }, + "order_seq": { + "type": "sequence" + }, + "product": {} + } +} diff --git a/examples/legacy_local/vschema_commerce_vsplit.json b/examples/legacy_local/vschema_commerce_vsplit.json new file mode 100644 index 00000000000..30f34172e31 --- /dev/null +++ b/examples/legacy_local/vschema_commerce_vsplit.json @@ -0,0 +1,5 @@ +{ + "tables": { + "product": {} + } +} diff --git a/examples/legacy_local/vschema_customer_sharded.json b/examples/legacy_local/vschema_customer_sharded.json new file mode 100644 index 00000000000..3109e2a2f3c --- /dev/null +++ b/examples/legacy_local/vschema_customer_sharded.json @@ -0,0 +1,34 @@ +{ + "sharded": true, + "vindexes": { + "hash": { + "type": "hash" + } + }, + "tables": { + "customer": { + "column_vindexes": [ + { + "column": "customer_id", + "name": "hash" + } + ], + "auto_increment": { + "column": "customer_id", + "sequence": "customer_seq" + } + }, + "corder": { + "column_vindexes": [ + { + "column": "customer_id", + "name": "hash" + } + ], + "auto_increment": { + "column": "order_id", + "sequence": "order_seq" + } + } + } +} diff --git a/examples/legacy_local/vschema_customer_vsplit.json b/examples/legacy_local/vschema_customer_vsplit.json new file mode 100644 index 00000000000..b113a9adc9e --- /dev/null +++ b/examples/legacy_local/vschema_customer_vsplit.json @@ -0,0 +1,6 @@ +{ + "tables": { + "customer": {}, + "corder": {} + } +} diff --git a/examples/local/201_customer_tablets.sh b/examples/local/201_customer_tablets.sh new file mode 100755 index 00000000000..0327ae30e6f --- /dev/null +++ b/examples/local/201_customer_tablets.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script creates the tablets and initializes them for vertical +# resharding it also splits the vschema between the two keyspaces +# old (commerce) and new (customer) + +source ./env.sh + +for i in 200 201 202; do + CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh + CELL=zone1 KEYSPACE=customer TABLET_UID=$i ./scripts/vttablet-up.sh +done + +vtctlclient -server localhost:15999 InitShardMaster -force customer/0 zone1-200 +vtctlclient -server localhost:15999 ApplyVSchema -vschema_file vschema_commerce_vsplit.json commerce +vtctlclient -server localhost:15999 ApplyVSchema -vschema_file vschema_customer_vsplit.json customer + diff --git a/examples/local/202_move_tables.sh b/examples/local/202_move_tables.sh new file mode 100755 index 00000000000..b7ffd9699bf --- /dev/null +++ b/examples/local/202_move_tables.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script copies over all the data from commerce keyspace to +# customer keyspace for the customer and corder tables + +source ./env.sh + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + MoveTables \ + -workflow=commerce2customer \ + commerce customer customer,corder + +sleep 2 diff --git a/examples/local/203_switch_reads.sh b/examples/local/203_switch_reads.sh new file mode 100755 index 00000000000..0b31909073f --- /dev/null +++ b/examples/local/203_switch_reads.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright 202- The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script migrates traffic for the new customer keyspace to the new +# tablets of types rdonly and replica + + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchReads \ + -tablet_type=rdonly \ + customer.commerce2customer + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchReads \ + -tablet_type=replica \ + customer.commerce2customer + diff --git a/examples/local/204_switch_writes.sh b/examples/local/204_switch_writes.sh new file mode 100755 index 00000000000..a0d5d3c31b5 --- /dev/null +++ b/examples/local/204_switch_writes.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script migrates master traffic for the customer keyspace to the +# new master tablet + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchWrites \ + customer.commerce2customer diff --git a/examples/local/205_clean_commerce.sh b/examples/local/205_clean_commerce.sh new file mode 100755 index 00000000000..8bccbe99b97 --- /dev/null +++ b/examples/local/205_clean_commerce.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script removes the customer and corder tables from the commerce +# keyspace + +vtctlclient -server localhost:15999 ApplySchema -sql-file drop_commerce_tables.sql commerce +vtctlclient -server localhost:15999 SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 rdonly +vtctlclient -server localhost:15999 SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 replica +vtctlclient -server localhost:15999 SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 master diff --git a/examples/local/302_new_shards.sh b/examples/local/302_new_shards.sh index c8830f97d0e..c6400bd0762 100755 --- a/examples/local/302_new_shards.sh +++ b/examples/local/302_new_shards.sh @@ -31,5 +31,3 @@ done vtctlclient -server localhost:15999 InitShardMaster -force customer/-80 zone1-300 vtctlclient -server localhost:15999 InitShardMaster -force customer/80- zone1-400 -vtctlclient -server localhost:15999 CopySchemaShard customer/0 customer/-80 -vtctlclient -server localhost:15999 CopySchemaShard customer/0 customer/80- diff --git a/examples/local/303_reshard.sh b/examples/local/303_reshard.sh new file mode 100755 index 00000000000..227ea3296b2 --- /dev/null +++ b/examples/local/303_reshard.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script copies the data from customer/0 to customer/-80 and customer/80- +# each row will be copied to exactly one shard based on the vindex value + +source ./env.sh + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + Reshard \ + customer.cust2cust "0" "-80,80-" + +sleep 2 \ No newline at end of file diff --git a/examples/local/304_switch_reads.sh b/examples/local/304_switch_reads.sh new file mode 100755 index 00000000000..aed6e6adada --- /dev/null +++ b/examples/local/304_switch_reads.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script migrates traffic for the rdonly and replica tablets + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchReads \ + -tablet_type=rdonly \ + customer.cust2cust + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchReads \ + -tablet_type=replica \ + customer.cust2cust diff --git a/examples/local/305_switch_writes.sh b/examples/local/305_switch_writes.sh new file mode 100755 index 00000000000..34dde1f971e --- /dev/null +++ b/examples/local/305_switch_writes.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script migrates traffic for the master tablet + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchWrites \ + customer.cust2cust + +# data has been copied over to shards, and databases for the new shards are now available + diff --git a/examples/local/401_teardown.sh b/examples/local/401_teardown.sh index d044b568afd..fb286ce5925 100755 --- a/examples/local/401_teardown.sh +++ b/examples/local/401_teardown.sh @@ -26,7 +26,9 @@ for tablet in 100 200 300 400; do # The zero tablet is up. Try to shutdown 0-2 tablet + mysqlctl for i in 0 1 2; do uid=$[$tablet + $i] + echo "Shutting down tablet zone1-$uid" CELL=zone1 TABLET_UID=$uid ./scripts/vttablet-down.sh + echo "Shutting down mysql zone1-$uid" CELL=zone1 TABLET_UID=$uid ./scripts/mysqlctl-down.sh done fi diff --git a/examples/local/scripts/mysqlctl-down.sh b/examples/local/scripts/mysqlctl-down.sh index 9e5491d1d4c..7965f026003 100755 --- a/examples/local/scripts/mysqlctl-down.sh +++ b/examples/local/scripts/mysqlctl-down.sh @@ -19,5 +19,5 @@ source ./env.sh -mysqlctl -tablet_uid $TABLET_UID shutdown +mysqlctl -tablet_uid $TABLET_UID shutdown diff --git a/examples/local/scripts/vtgate-up.sh b/examples/local/scripts/vtgate-up.sh index e3fe2d17f82..03501776f4d 100755 --- a/examples/local/scripts/vtgate-up.sh +++ b/examples/local/scripts/vtgate-up.sh @@ -41,6 +41,7 @@ vtgate \ -service_map 'grpc-vtgateservice' \ -pid_file $VTDATAROOT/tmp/vtgate.pid \ -mysql_auth_server_impl none \ + -tablet_refresh_interval 5s \ > $VTDATAROOT/tmp/vtgate.out 2>&1 & # Block waiting for vtgate to be listening diff --git a/examples/local/topo-etcd2.sh b/examples/local/topo-etcd2.sh index c61543a806c..4b6ebc62fc4 100644 --- a/examples/local/topo-etcd2.sh +++ b/examples/local/topo-etcd2.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/local/topo-zk2.sh b/examples/local/topo-zk2.sh index 29380949d8f..1a7cedfe2cd 100644 --- a/examples/local/topo-zk2.sh +++ b/examples/local/topo-zk2.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/legacy_local_example.sh b/test/legacy_local_example.sh new file mode 100755 index 00000000000..b2f5c16189c --- /dev/null +++ b/test/legacy_local_example.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This test runs through the scripts in examples/local to make sure they work. +# It should be kept in sync with the steps in https://vitess.io/docs/get-started/local/ +# So we can detect if a regression affecting a tutorial is introduced. + +source build.env + +set -xe + +cd "$VTROOT/examples/legacy_local" + +unset VTROOT # ensure that the examples can run without VTROOT now. + +./101_initial_cluster.sh + +mysql -h 127.0.0.1 -P 15306 < ../common/insert_commerce_data.sql +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_commerce_data.sql +./201_customer_keyspace.sh +./202_customer_tablets.sh +./203_vertical_split.sh +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer0_data.sql + +./204_vertical_migrate_replicas.sh +./205_vertical_migrate_master.sh +# Expected to fail! +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_commerce_data.sql || echo "Blacklist working as expected" +./206_clean_commerce.sh +# Expected to fail! +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_commerce_data.sql || echo "Tables missing as expected" + +./301_customer_sharded.sh +./302_new_shards.sh + +# Wait for the schema to be targetable before proceeding +# TODO: Eliminate this race in the examples' scripts +for shard in "customer/-80" "customer/80-"; do + while true; do + mysql -h 127.0.0.1 -P 15306 "$shard" -e 'show tables' && break || echo "waiting for shard: $shard!" + sleep 1 + done; +done; + +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer-80_data.sql +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer80-_data.sql + +./303_horizontal_split.sh + +./304_migrate_replicas.sh +./305_migrate_master.sh + +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer-80_data.sql + +./401_teardown.sh + diff --git a/test/local_example.sh b/test/local_example.sh index 71c2042fbd7..ee5cd0142ec 100755 --- a/test/local_example.sh +++ b/test/local_example.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -30,19 +30,30 @@ unset VTROOT # ensure that the examples can run without VTROOT now. mysql -h 127.0.0.1 -P 15306 < ../common/insert_commerce_data.sql mysql -h 127.0.0.1 -P 15306 --table < ../common/select_commerce_data.sql -./201_customer_keyspace.sh -./202_customer_tablets.sh -./203_vertical_split.sh -mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer0_data.sql -./204_vertical_migrate_replicas.sh -./205_vertical_migrate_master.sh +./201_customer_tablets.sh + +for shard in "customer/0"; do + while true; do + mysql -h 127.0.0.1 -P 15306 "$shard" -e 'show tables' && break || echo "waiting for shard: $shard!" + sleep 1 + done; +done; + +./202_move_tables.sh + +./203_switch_reads.sh + +./204_switch_writes.sh + +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer0_data.sql # Expected to fail! mysql -h 127.0.0.1 -P 15306 --table < ../common/select_commerce_data.sql || echo "Blacklist working as expected" -./206_clean_commerce.sh +./205_clean_commerce.sh # Expected to fail! mysql -h 127.0.0.1 -P 15306 --table < ../common/select_commerce_data.sql || echo "Tables missing as expected" + ./301_customer_sharded.sh ./302_new_shards.sh @@ -55,15 +66,13 @@ for shard in "customer/-80" "customer/80-"; do done; done; -mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer-80_data.sql -mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer80-_data.sql +./303_reshard.sh -./303_horizontal_split.sh - -./304_migrate_replicas.sh -./305_migrate_master.sh +./304_switch_reads.sh +./305_switch_writes.sh mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer-80_data.sql +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer80-_data.sql ./401_teardown.sh