From e2ffdb228dffb89f0a0587452eb93656936a2310 Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 6 Dec 2018 12:45:09 -0800 Subject: [PATCH 1/7] add descriptive comments to scripts, add etcd option, set MYSQL_FLAVOR is it isn't already set Signed-off-by: deepthi --- examples/local/101_initial_cluster.sh | 21 +++++++++++++++++-- examples/local/201_customer_keyspace.sh | 3 +-- examples/local/202_customer_tablets.sh | 5 +++-- examples/local/203_vertical_split.sh | 4 ++-- .../local/204_vertical_migrate_replicas.sh | 4 ++-- examples/local/205_vertical_migrate_master.sh | 4 ++-- examples/local/206_clean_commerce.sh | 4 ++-- examples/local/301_customer_sharded.sh | 7 +++++-- examples/local/302_new_shards.sh | 4 ++-- examples/local/303_horizontal_split.sh | 4 ++-- examples/local/304_migrate_replicas.sh | 3 +-- examples/local/305_migrate_master.sh | 3 +-- examples/local/306_down_shard_0.sh | 3 +-- examples/local/401_teardown.sh | 13 +++++++++--- examples/local/env.sh | 9 ++++++++ examples/local/etcd-down.sh | 2 +- examples/local/etcd-up.sh | 6 +++--- examples/local/vttablet-up.sh | 4 +--- 18 files changed, 67 insertions(+), 36 deletions(-) diff --git a/examples/local/101_initial_cluster.sh b/examples/local/101_initial_cluster.sh index a325f9ad2a0..921afde70fb 100755 --- a/examples/local/101_initial_cluster.sh +++ b/examples/local/101_initial_cluster.sh @@ -14,20 +14,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components +# this script brings up zookeeper and all the vitess components # required for a single shard deployment. set -e script_root=`dirname "${BASH_SOURCE}"` -CELL=zone1 $script_root/zk-up.sh +# start topo server +if [ "${TOPO}" = "etcd2" ]; then + CELL=zone1 $script_root/etcd-up.sh +else + CELL=zone1 $script_root/zk-up.sh +fi + +# start vtctld CELL=zone1 $script_root/vtctld-up.sh + +# start vttablets for keyspace commerce CELL=zone1 KEYSPACE=commerce UID_BASE=100 $script_root/vttablet-up.sh sleep 15 + +# set one of the replicas to master ./lvtctl.sh InitShardMaster -force commerce/0 zone1-100 + +# create the schema ./lvtctl.sh ApplySchema -sql-file create_commerce_schema.sql commerce + +# create the vschema ./lvtctl.sh ApplyVSchema -vschema_file vschema_commerce_initial.json commerce + +# start vtgate CELL=zone1 $script_root/vtgate-up.sh disown -a diff --git a/examples/local/201_customer_keyspace.sh b/examples/local/201_customer_keyspace.sh index 0aedba25fe7..210c4eaf5a9 100755 --- a/examples/local/201_customer_keyspace.sh +++ b/examples/local/201_customer_keyspace.sh @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script creates a new keyspace in preparation for vertical resharding set -e diff --git a/examples/local/202_customer_tablets.sh b/examples/local/202_customer_tablets.sh index 235b76fd4f0..75deccbacb6 100755 --- a/examples/local/202_customer_tablets.sh +++ b/examples/local/202_customer_tablets.sh @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script creates the tablets and initializes them for vertical +# resharding it also splits the vschema between the two keyspaces +# old (commerce) and new (customer) set -e diff --git a/examples/local/203_vertical_split.sh b/examples/local/203_vertical_split.sh index 053a97f7b38..7a88c0d4860 100755 --- a/examples/local/203_vertical_split.sh +++ b/examples/local/203_vertical_split.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script copies over all the data from commerce keyspace to +# customer keyspace for the customer and corder tables set -e diff --git a/examples/local/204_vertical_migrate_replicas.sh b/examples/local/204_vertical_migrate_replicas.sh index 4c9acc38445..567dca6f508 100755 --- a/examples/local/204_vertical_migrate_replicas.sh +++ b/examples/local/204_vertical_migrate_replicas.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script migrates traffic for the new customer keyspace to the new +# tablets of types rdonly and replica set -e diff --git a/examples/local/205_vertical_migrate_master.sh b/examples/local/205_vertical_migrate_master.sh index a21c1bf6848..560564e5099 100755 --- a/examples/local/205_vertical_migrate_master.sh +++ b/examples/local/205_vertical_migrate_master.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script migrates master traffic for the customer keyspace to the +# new master tablet set -e diff --git a/examples/local/206_clean_commerce.sh b/examples/local/206_clean_commerce.sh index 010bd8ba999..eeef5d2c04f 100755 --- a/examples/local/206_clean_commerce.sh +++ b/examples/local/206_clean_commerce.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script removes the customer and corder tables from the commerce +# keyspace set -e diff --git a/examples/local/301_customer_sharded.sh b/examples/local/301_customer_sharded.sh index a1774158b30..c7f766810d6 100755 --- a/examples/local/301_customer_sharded.sh +++ b/examples/local/301_customer_sharded.sh @@ -14,8 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script creates vitess sequences for the auto_increment fields +# and alters the fields to no longer be auto_increment in preparation +# for horizontal sharding +# it also changes the customer vschema from unsharded to sharded and +# sets up the necessary vindexes set -e diff --git a/examples/local/302_new_shards.sh b/examples/local/302_new_shards.sh index de480e1494e..30c1cb6b507 100755 --- a/examples/local/302_new_shards.sh +++ b/examples/local/302_new_shards.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script brings up new tablets for the two new shards that we will +# be creating in the customer keyspace and copies the schema set -e diff --git a/examples/local/303_horizontal_split.sh b/examples/local/303_horizontal_split.sh index 8859b5dcf0a..6851f6c378d 100755 --- a/examples/local/303_horizontal_split.sh +++ b/examples/local/303_horizontal_split.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script copies the data from customer/0 to customer/-80 and customer/80- +# each row will be copied to exactly one shard based on the vindex value set -e diff --git a/examples/local/304_migrate_replicas.sh b/examples/local/304_migrate_replicas.sh index 6722e21e2f2..9e5d381e515 100755 --- a/examples/local/304_migrate_replicas.sh +++ b/examples/local/304_migrate_replicas.sh @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script migrates traffic for the rdonly and replica tablets set -e diff --git a/examples/local/305_migrate_master.sh b/examples/local/305_migrate_master.sh index ede4e3c970f..a9a01006171 100755 --- a/examples/local/305_migrate_master.sh +++ b/examples/local/305_migrate_master.sh @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script migrates traffic for the master tablet set -e diff --git a/examples/local/306_down_shard_0.sh b/examples/local/306_down_shard_0.sh index e8fe8112ddf..5d32383dea4 100755 --- a/examples/local/306_down_shard_0.sh +++ b/examples/local/306_down_shard_0.sh @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script deletes the old shard 0 which has been replaced by 2 shards set -e diff --git a/examples/local/401_teardown.sh b/examples/local/401_teardown.sh index dc58874c4f6..d0663ac7da4 100755 --- a/examples/local/401_teardown.sh +++ b/examples/local/401_teardown.sh @@ -14,8 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this scripts brings up zookeeper and all the vitess components -# required for a single shard deployment. +# this script brings down zookeeper and all the vitess components +# we brought up in the example +# optionally, you may want to delete everything that was created +# by the example from $VTDATAROOT set -e @@ -27,6 +29,11 @@ CELL=zone1 UID_BASE=200 $script_root/vttablet-down.sh CELL=zone1 UID_BASE=300 $script_root/vttablet-down.sh CELL=zone1 UID_BASE=400 $script_root/vttablet-down.sh ./vtctld-down.sh -./zk-down.sh + +if [ "${TOPO}" = "etcd2" ]; then + CELL=zone1 $script_root/etcd-down.sh +else + CELL=zone1 $script_root/zk-down.sh +fi disown -a diff --git a/examples/local/env.sh b/examples/local/env.sh index 00ab9bacdaa..dab3ae00be6 100644 --- a/examples/local/env.sh +++ b/examples/local/env.sh @@ -28,6 +28,15 @@ if [ -z "$VT_MYSQL_ROOT" ]; then export VT_MYSQL_ROOT=$(dirname `dirname $mysql_path`) fi +# restore MYSQL_FLAVOR, saved by bootstrap.sh +if [ -r "$VTROOT/dist/MYSQL_FLAVOR" ]; then + MYSQL_FLAVOR=$(cat "$VTROOT/dist/MYSQL_FLAVOR") + export MYSQL_FLAVOR +fi + +if [ -z "$MYSQL_FLAVOR" ]; then + export MYSQL_FLAVOR=MySQL56 +fi if [ "${TOPO}" = "etcd2" ]; then echo "enter etcd2 env" diff --git a/examples/local/etcd-down.sh b/examples/local/etcd-down.sh index 5888ddb52e8..5bc10ac0cf4 100755 --- a/examples/local/etcd-down.sh +++ b/examples/local/etcd-down.sh @@ -26,5 +26,5 @@ source "${script_root}/env.sh" # Stop etcd servers. echo "Stopping etcd servers..." -ETCD_VERSION=$(cat "$VTROOT/dist/etcd/version") +ETCD_VERSION=$(cat "$VTROOT/dist/etcd/.installed_version") kill -9 "$(pgrep -f "dist/etcd/etcd-${ETCD_VERSION}-linux-amd64/etcd")" diff --git a/examples/local/etcd-up.sh b/examples/local/etcd-up.sh index 7d68dae9f27..955169e88fd 100755 --- a/examples/local/etcd-up.sh +++ b/examples/local/etcd-up.sh @@ -25,9 +25,9 @@ script_root=$(dirname "${BASH_SOURCE[0]}") # shellcheck disable=SC1091 source "${script_root}/env.sh" -ETCD_VERSION=$(cat "${VTROOT}/dist/etcd/version") +ETCD_VERSION=$(cat "${VTROOT}/dist/etcd/.installed_version") -"${VTROOT}"/dist/etcd/etcd-"${ETCD_VERSION}"-linux-amd64/etcd --data-dir "${VTDATAROOT}/etcd/" --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" & +"${VTROOT}"/dist/etcd/etcd-"${ETCD_VERSION}"-linux-amd64/etcd --data-dir "${VTDATAROOT}/etcd/" --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" > "${VTDATAROOT}"/tmp/etcd.out 2>&1 & sleep 5 echo "add /vitess/global" @@ -37,7 +37,7 @@ echo "add /vitess/global" echo "add /vitess/$cell" "${VTROOT}"/dist/etcd/etcd-"${ETCD_VERSION}"-linux-amd64/etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/$cell & -# And also add the CellInfo description for the 'test' cell. +# And also add the CellInfo description for the cell. # If the node already exists, it's fine, means we used existing data. echo "add $cell CellInfo" set +e diff --git a/examples/local/vttablet-up.sh b/examples/local/vttablet-up.sh index b5ccd4a2e43..00cea50a5b5 100755 --- a/examples/local/vttablet-up.sh +++ b/examples/local/vttablet-up.sh @@ -43,11 +43,9 @@ export EXTRA_MY_CNF=$VTROOT/config/mycnf/default-fast.cnf case "$MYSQL_FLAVOR" in "MySQL56") export EXTRA_MY_CNF=$EXTRA_MY_CNF:$VTROOT/config/mycnf/master_mysql56.cnf -# export EXTRA_MY_CNF=$VTROOT/config/mycnf/master_mysql56.cnf ;; "MariaDB") export EXTRA_MY_CNF=$EXTRA_MY_CNF:$VTROOT/config/mycnf/master_mariadb.cnf -# export EXTRA_MY_CNF=$VTROOT/config/mycnf/master_mariadb.cnf ;; *) echo "Please set MYSQL_FLAVOR to MySQL56 or MariaDB." @@ -57,7 +55,7 @@ esac mkdir -p $VTDATAROOT/backups -# Start 5 vttablets by default. +# Start 3 vttablets by default. # Pass TABLETS_UIDS indices as env variable to change uids=${TABLETS_UIDS:-'0 1 2'} From c16933eb5f7b0fc1b793588be7d8180b86f8e20e Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 6 Dec 2018 16:27:52 -0800 Subject: [PATCH 2/7] remove unused files, add sql scripts to show intermediate and final results of resharding Signed-off-by: deepthi --- examples/local/101_initial_cluster.sh | 5 ++ examples/local/203_vertical_split.sh | 4 + examples/local/205_vertical_migrate_master.sh | 3 + examples/local/206_clean_commerce.sh | 3 + examples/local/302_new_shards.sh | 5 ++ examples/local/305_migrate_master.sh | 4 + examples/local/client.py | 86 ------------------- examples/local/client.sh | 33 ------- examples/local/create_test_table.sql | 7 -- examples/local/insert_commerce_data.sql | 12 +++ examples/local/select_commerce_data.sql | 8 ++ examples/local/select_customer-80_data.sql | 6 ++ examples/local/select_customer0_data.sql | 6 ++ examples/local/select_customer80-_data.sql | 6 ++ examples/local/vschema.json | 18 ---- 15 files changed, 62 insertions(+), 144 deletions(-) delete mode 100644 examples/local/client.py delete mode 100755 examples/local/client.sh delete mode 100644 examples/local/create_test_table.sql create mode 100644 examples/local/insert_commerce_data.sql create mode 100644 examples/local/select_commerce_data.sql create mode 100644 examples/local/select_customer-80_data.sql create mode 100644 examples/local/select_customer0_data.sql create mode 100644 examples/local/select_customer80-_data.sql delete mode 100644 examples/local/vschema.json diff --git a/examples/local/101_initial_cluster.sh b/examples/local/101_initial_cluster.sh index 921afde70fb..74f87d52396 100755 --- a/examples/local/101_initial_cluster.sh +++ b/examples/local/101_initial_cluster.sh @@ -46,5 +46,10 @@ sleep 15 # start vtgate CELL=zone1 $script_root/vtgate-up.sh +sleep 5 + +# insert rows into the tables. we are actually connecting to vtgate here, it will route the sql correctly +mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password < insert_commerce_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_commerce_data.sql disown -a diff --git a/examples/local/203_vertical_split.sh b/examples/local/203_vertical_split.sh index 7a88c0d4860..bf3630e9a30 100755 --- a/examples/local/203_vertical_split.sh +++ b/examples/local/203_vertical_split.sh @@ -30,4 +30,8 @@ $VTROOT/bin/vtworker \ -use_v3_resharding_mode \ VerticalSplitClone -min_healthy_rdonly_tablets=1 -tables=customer,corder customer/0 +sleep 2 +# check that customer and order data has been copied to customer keyspace +mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_customer0_data.sql + disown -a diff --git a/examples/local/205_vertical_migrate_master.sh b/examples/local/205_vertical_migrate_master.sh index 560564e5099..0578ddbaaff 100755 --- a/examples/local/205_vertical_migrate_master.sh +++ b/examples/local/205_vertical_migrate_master.sh @@ -23,4 +23,7 @@ script_root=`dirname "${BASH_SOURCE}"` ./lvtctl.sh MigrateServedFrom customer/0 master +# customer and order data cannot be accessed from commerce keyspace +mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_commerce_data.sql + disown -a diff --git a/examples/local/206_clean_commerce.sh b/examples/local/206_clean_commerce.sh index eeef5d2c04f..342192de960 100755 --- a/examples/local/206_clean_commerce.sh +++ b/examples/local/206_clean_commerce.sh @@ -26,4 +26,7 @@ script_root=`dirname "${BASH_SOURCE}"` ./lvtctl.sh SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 replica ./lvtctl.sh SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 master +# customer and order tables have now been deleted from commerce keyspace +mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_commerce_data.sql + disown -a diff --git a/examples/local/302_new_shards.sh b/examples/local/302_new_shards.sh index 30c1cb6b507..e11caff88ef 100755 --- a/examples/local/302_new_shards.sh +++ b/examples/local/302_new_shards.sh @@ -29,4 +29,9 @@ sleep 15 ./lvtctl.sh CopySchemaShard customer/0 customer/-80 ./lvtctl.sh CopySchemaShard customer/0 customer/80- +sleep 15 +# new tables are empty +mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_customer-80_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_customer80-_data.sql + disown -a diff --git a/examples/local/305_migrate_master.sh b/examples/local/305_migrate_master.sh index a9a01006171..79c6ea22264 100755 --- a/examples/local/305_migrate_master.sh +++ b/examples/local/305_migrate_master.sh @@ -22,4 +22,8 @@ script_root=`dirname "${BASH_SOURCE}"` ./lvtctl.sh MigrateServedTypes customer/0 master +# data has been copied over to shards, and databases for the new shards are now available +mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_customer-80_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_customer80-_data.sql + disown -a diff --git a/examples/local/client.py b/examples/local/client.py deleted file mode 100644 index 1c7e8029b98..00000000000 --- a/examples/local/client.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Sample Vitess client in Python. - -This is a sample for using the Python Vitess client. -It's a script that inserts some random messages on random pages of the -guestbook sample app. - -Before running this, start up a local example cluster as described in the -README.md file. - -Then run client.sh, which sets up PYTHONPATH before running client.py: -vitess/examples/local$ ./client.sh -""" - -import argparse -import random -import time - -from vtdb import vtgate_client - -# register the python gRPC client upon import -from vtdb import grpc_vtgate_client # pylint: disable=unused-import - -# Parse args -parser = argparse.ArgumentParser() -parser.add_argument('--server', dest='server', default='localhost:15991') -parser.add_argument('--timeout', dest='timeout', type=float, default='10.0') -args = parser.parse_args() - -# Connect -conn = vtgate_client.connect('grpc', args.server, args.timeout) - -try: - # Insert some messages on random pages. - print 'Inserting into master...' - cursor = conn.cursor(tablet_type='master', writable=True) - for i in range(3): - page = random.randint(1, 100) - - cursor.begin() - cursor.execute( - 'INSERT INTO messages (page, time_created_ns, message)' - ' VALUES (:page, :time_created_ns, :message)', - { - 'page': page, - 'time_created_ns': int(time.time() * 1e9), - 'message': 'V is for speed', - }) - cursor.commit() - - # Read it back from the master. - print 'Reading from master...' - cursor.execute('SELECT page, time_created_ns, message FROM messages', {}) - for row in cursor.fetchall(): - print row - - cursor.close() - - # Read from a replica. - # Note that this may be behind master due to replication lag. - print 'Reading from replica...' - cursor = conn.cursor(tablet_type='replica') - cursor.execute('SELECT page, time_created_ns, message FROM messages', {}) - for row in cursor.fetchall(): - print row - cursor.close() - -finally: - # Clean up - conn.close() diff --git a/examples/local/client.sh b/examples/local/client.sh deleted file mode 100755 index fd7a6eeb0db..00000000000 --- a/examples/local/client.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is a wrapper script that sets up the environment for client.py. - -set -e - -hostname=`hostname -f` - -# We expect to find zk-client-conf.json in the same folder as this script. -script_root=`dirname "${BASH_SOURCE}"` - -# Set up environment. -for pkg in `find $VTROOT/dist -name site-packages`; do - export PYTHONPATH=$pkg:$PYTHONPATH -done - -export PYTHONPATH=$VTROOT/py-vtdb:$PYTHONPATH - -exec env python $script_root/client.py $* diff --git a/examples/local/create_test_table.sql b/examples/local/create_test_table.sql deleted file mode 100644 index bfb24b766ad..00000000000 --- a/examples/local/create_test_table.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE messages ( - page BIGINT(20) UNSIGNED, - time_created_ns BIGINT(20) UNSIGNED, - message VARCHAR(10000), - PRIMARY KEY (page, time_created_ns) -) ENGINE=InnoDB - diff --git a/examples/local/insert_commerce_data.sql b/examples/local/insert_commerce_data.sql new file mode 100644 index 00000000000..32c4ab332fd --- /dev/null +++ b/examples/local/insert_commerce_data.sql @@ -0,0 +1,12 @@ +insert into customer(email) values('alice@domain.com'); +insert into customer(email) values('bob@domain.com'); +insert into customer(email) values('charlie@domain.com'); +insert into customer(email) values('dan@domain.com'); +insert into customer(email) values('eve@domain.com'); +insert into product(sku, description, price) values('SKU-1001', 'Monitor', 100); +insert into product(sku, description, price) values('SKU-1002', 'Keyboard', 30); +insert into corder(customer_id, sku, price) values(1, 'SKU-1001', 100); +insert into corder(customer_id, sku, price) values(2, 'SKU-1002', 30); +insert into corder(customer_id, sku, price) values(3, 'SKU-1002', 30); +insert into corder(customer_id, sku, price) values(4, 'SKU-1002', 30); +insert into corder(customer_id, sku, price) values(5, 'SKU-1002', 30); diff --git a/examples/local/select_commerce_data.sql b/examples/local/select_commerce_data.sql new file mode 100644 index 00000000000..d7494950b2d --- /dev/null +++ b/examples/local/select_commerce_data.sql @@ -0,0 +1,8 @@ +\! echo 'Using commerce/0' +use commerce/0; +\! echo 'Customer' +select * from customer; +\! echo 'Product' +select * from product; +\! echo 'COrder' +select * from corder; diff --git a/examples/local/select_customer-80_data.sql b/examples/local/select_customer-80_data.sql new file mode 100644 index 00000000000..2155068b423 --- /dev/null +++ b/examples/local/select_customer-80_data.sql @@ -0,0 +1,6 @@ +\! echo 'Using customer/-80' +use customer/-80; +\! echo 'Customer' +select * from customer; +\! echo 'COrder' +select * from corder; diff --git a/examples/local/select_customer0_data.sql b/examples/local/select_customer0_data.sql new file mode 100644 index 00000000000..01b0548ce3f --- /dev/null +++ b/examples/local/select_customer0_data.sql @@ -0,0 +1,6 @@ +\! echo 'Using customer/0' +use customer/0; +\! echo 'Customer' +select * from customer; +\! echo 'COrder' +select * from corder; diff --git a/examples/local/select_customer80-_data.sql b/examples/local/select_customer80-_data.sql new file mode 100644 index 00000000000..9e4a3c95dea --- /dev/null +++ b/examples/local/select_customer80-_data.sql @@ -0,0 +1,6 @@ +\! echo 'Using customer/80-' +use customer/80-; +\! echo 'Customer' +select * from customer; +\! echo 'COrder' +select * from corder; diff --git a/examples/local/vschema.json b/examples/local/vschema.json deleted file mode 100644 index 17e5dedf0c8..00000000000 --- a/examples/local/vschema.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "sharded": true, - "vindexes": { - "hash": { - "type": "hash" - } - }, - "tables": { - "messages": { - "column_vindexes": [ - { - "column": "page", - "name": "hash" - } - ] - } - } -} From 66426a95ed311a1ee20d00c55e425249ee742c7e Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 6 Dec 2018 18:06:39 -0800 Subject: [PATCH 3/7] add back files required for test Signed-off-by: deepthi --- examples/local/client.py | 86 ++++++++++++++++++++++++++++ examples/local/client.sh | 33 +++++++++++ examples/local/create_test_table.sql | 7 +++ 3 files changed, 126 insertions(+) create mode 100644 examples/local/client.py create mode 100755 examples/local/client.sh create mode 100644 examples/local/create_test_table.sql diff --git a/examples/local/client.py b/examples/local/client.py new file mode 100644 index 00000000000..1c7e8029b98 --- /dev/null +++ b/examples/local/client.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Sample Vitess client in Python. + +This is a sample for using the Python Vitess client. +It's a script that inserts some random messages on random pages of the +guestbook sample app. + +Before running this, start up a local example cluster as described in the +README.md file. + +Then run client.sh, which sets up PYTHONPATH before running client.py: +vitess/examples/local$ ./client.sh +""" + +import argparse +import random +import time + +from vtdb import vtgate_client + +# register the python gRPC client upon import +from vtdb import grpc_vtgate_client # pylint: disable=unused-import + +# Parse args +parser = argparse.ArgumentParser() +parser.add_argument('--server', dest='server', default='localhost:15991') +parser.add_argument('--timeout', dest='timeout', type=float, default='10.0') +args = parser.parse_args() + +# Connect +conn = vtgate_client.connect('grpc', args.server, args.timeout) + +try: + # Insert some messages on random pages. + print 'Inserting into master...' + cursor = conn.cursor(tablet_type='master', writable=True) + for i in range(3): + page = random.randint(1, 100) + + cursor.begin() + cursor.execute( + 'INSERT INTO messages (page, time_created_ns, message)' + ' VALUES (:page, :time_created_ns, :message)', + { + 'page': page, + 'time_created_ns': int(time.time() * 1e9), + 'message': 'V is for speed', + }) + cursor.commit() + + # Read it back from the master. + print 'Reading from master...' + cursor.execute('SELECT page, time_created_ns, message FROM messages', {}) + for row in cursor.fetchall(): + print row + + cursor.close() + + # Read from a replica. + # Note that this may be behind master due to replication lag. + print 'Reading from replica...' + cursor = conn.cursor(tablet_type='replica') + cursor.execute('SELECT page, time_created_ns, message FROM messages', {}) + for row in cursor.fetchall(): + print row + cursor.close() + +finally: + # Clean up + conn.close() diff --git a/examples/local/client.sh b/examples/local/client.sh new file mode 100755 index 00000000000..fd7a6eeb0db --- /dev/null +++ b/examples/local/client.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a wrapper script that sets up the environment for client.py. + +set -e + +hostname=`hostname -f` + +# We expect to find zk-client-conf.json in the same folder as this script. +script_root=`dirname "${BASH_SOURCE}"` + +# Set up environment. +for pkg in `find $VTROOT/dist -name site-packages`; do + export PYTHONPATH=$pkg:$PYTHONPATH +done + +export PYTHONPATH=$VTROOT/py-vtdb:$PYTHONPATH + +exec env python $script_root/client.py $* diff --git a/examples/local/create_test_table.sql b/examples/local/create_test_table.sql new file mode 100644 index 00000000000..bfb24b766ad --- /dev/null +++ b/examples/local/create_test_table.sql @@ -0,0 +1,7 @@ +CREATE TABLE messages ( + page BIGINT(20) UNSIGNED, + time_created_ns BIGINT(20) UNSIGNED, + message VARCHAR(10000), + PRIMARY KEY (page, time_created_ns) +) ENGINE=InnoDB + From 06cb4f4ab7ed284873ec3db5af0ec9f20f96958d Mon Sep 17 00:00:00 2001 From: deepthi Date: Fri, 7 Dec 2018 10:14:46 -0800 Subject: [PATCH 4/7] use auth=none for vtgate, move sql scripts to common location Signed-off-by: deepthi --- examples/{local => common}/insert_commerce_data.sql | 0 examples/{local => common}/select_commerce_data.sql | 0 examples/{local => common}/select_customer-80_data.sql | 0 examples/{local => common}/select_customer0_data.sql | 0 examples/{local => common}/select_customer80-_data.sql | 0 examples/local/101_initial_cluster.sh | 4 ++-- examples/local/203_vertical_split.sh | 2 +- examples/local/205_vertical_migrate_master.sh | 2 +- examples/local/206_clean_commerce.sh | 2 +- examples/local/302_new_shards.sh | 4 ++-- examples/local/305_migrate_master.sh | 4 ++-- examples/local/vtgate-up.sh | 9 +++++++-- 12 files changed, 16 insertions(+), 11 deletions(-) rename examples/{local => common}/insert_commerce_data.sql (100%) rename examples/{local => common}/select_commerce_data.sql (100%) rename examples/{local => common}/select_customer-80_data.sql (100%) rename examples/{local => common}/select_customer0_data.sql (100%) rename examples/{local => common}/select_customer80-_data.sql (100%) diff --git a/examples/local/insert_commerce_data.sql b/examples/common/insert_commerce_data.sql similarity index 100% rename from examples/local/insert_commerce_data.sql rename to examples/common/insert_commerce_data.sql diff --git a/examples/local/select_commerce_data.sql b/examples/common/select_commerce_data.sql similarity index 100% rename from examples/local/select_commerce_data.sql rename to examples/common/select_commerce_data.sql diff --git a/examples/local/select_customer-80_data.sql b/examples/common/select_customer-80_data.sql similarity index 100% rename from examples/local/select_customer-80_data.sql rename to examples/common/select_customer-80_data.sql diff --git a/examples/local/select_customer0_data.sql b/examples/common/select_customer0_data.sql similarity index 100% rename from examples/local/select_customer0_data.sql rename to examples/common/select_customer0_data.sql diff --git a/examples/local/select_customer80-_data.sql b/examples/common/select_customer80-_data.sql similarity index 100% rename from examples/local/select_customer80-_data.sql rename to examples/common/select_customer80-_data.sql diff --git a/examples/local/101_initial_cluster.sh b/examples/local/101_initial_cluster.sh index 74f87d52396..1c30fdff73f 100755 --- a/examples/local/101_initial_cluster.sh +++ b/examples/local/101_initial_cluster.sh @@ -49,7 +49,7 @@ CELL=zone1 $script_root/vtgate-up.sh sleep 5 # insert rows into the tables. we are actually connecting to vtgate here, it will route the sql correctly -mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password < insert_commerce_data.sql -mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_commerce_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user < $script_root/../common/insert_commerce_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_commerce_data.sql disown -a diff --git a/examples/local/203_vertical_split.sh b/examples/local/203_vertical_split.sh index bf3630e9a30..21afb3798c2 100755 --- a/examples/local/203_vertical_split.sh +++ b/examples/local/203_vertical_split.sh @@ -32,6 +32,6 @@ $VTROOT/bin/vtworker \ sleep 2 # check that customer and order data has been copied to customer keyspace -mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_customer0_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_customer0_data.sql disown -a diff --git a/examples/local/205_vertical_migrate_master.sh b/examples/local/205_vertical_migrate_master.sh index 0578ddbaaff..d2024c05739 100755 --- a/examples/local/205_vertical_migrate_master.sh +++ b/examples/local/205_vertical_migrate_master.sh @@ -24,6 +24,6 @@ script_root=`dirname "${BASH_SOURCE}"` ./lvtctl.sh MigrateServedFrom customer/0 master # customer and order data cannot be accessed from commerce keyspace -mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_commerce_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_commerce_data.sql disown -a diff --git a/examples/local/206_clean_commerce.sh b/examples/local/206_clean_commerce.sh index 342192de960..2b410a5d12e 100755 --- a/examples/local/206_clean_commerce.sh +++ b/examples/local/206_clean_commerce.sh @@ -27,6 +27,6 @@ script_root=`dirname "${BASH_SOURCE}"` ./lvtctl.sh SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 master # customer and order tables have now been deleted from commerce keyspace -mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_commerce_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_commerce_data.sql disown -a diff --git a/examples/local/302_new_shards.sh b/examples/local/302_new_shards.sh index e11caff88ef..c6909d04d6d 100755 --- a/examples/local/302_new_shards.sh +++ b/examples/local/302_new_shards.sh @@ -31,7 +31,7 @@ sleep 15 sleep 15 # new tables are empty -mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_customer-80_data.sql -mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_customer80-_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_customer-80_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_customer80-_data.sql disown -a diff --git a/examples/local/305_migrate_master.sh b/examples/local/305_migrate_master.sh index 79c6ea22264..5f989708f9f 100755 --- a/examples/local/305_migrate_master.sh +++ b/examples/local/305_migrate_master.sh @@ -23,7 +23,7 @@ script_root=`dirname "${BASH_SOURCE}"` ./lvtctl.sh MigrateServedTypes customer/0 master # data has been copied over to shards, and databases for the new shards are now available -mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_customer-80_data.sql -mysql -h 127.0.0.1 -P 15306 -u mysql_user -pmysql_password --table < select_customer80-_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_customer-80_data.sql +mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_customer80-_data.sql disown -a diff --git a/examples/local/vtgate-up.sh b/examples/local/vtgate-up.sh index f79a4d9dc6d..77483499e71 100755 --- a/examples/local/vtgate-up.sh +++ b/examples/local/vtgate-up.sh @@ -54,13 +54,19 @@ then optional_tls_args="-grpc_cert $cert_dir/server-cert.pem -grpc_key $cert_dir/server-key.pem -grpc_ca $cert_dir/ca-cert.pem" fi -optional_auth_args='' +optional_auth_args='-mysql_auth_server_impl none' if [ "$1" = "--enable-grpc-static-auth" ]; then echo "Enabling Auth with static authentication in grpc" optional_auth_args='-grpc_auth_static_client_creds ./grpc_static_client_auth.json' fi +if [ "$1" = "--enable-mysql-static-auth" ]; +then + echo "Enabling Auth with mysql static authentication" + optional_auth_args='-mysql_auth_server_static_file ./mysql_auth_server_static_creds.json' +fi + # Start vtgate. # shellcheck disable=SC2086 $VTROOT/bin/vtgate \ @@ -71,7 +77,6 @@ $VTROOT/bin/vtgate \ -grpc_port $grpc_port \ -mysql_server_port $mysql_server_port \ -mysql_server_socket_path $mysql_server_socket_path \ - -mysql_auth_server_static_file "./mysql_auth_server_static_creds.json" \ -cell $cell \ -cells_to_watch $cell \ -tablet_types_to_wait MASTER,REPLICA \ From 2be922a20c4d44ec35b20c0881dd7a2736afc883 Mon Sep 17 00:00:00 2001 From: deepthi Date: Fri, 7 Dec 2018 11:16:37 -0800 Subject: [PATCH 5/7] add mysql helper script to local example, remove calls to mysql from scripts, they will be executed by the user using lmysql.sh Signed-off-by: deepthi --- examples/local/101_initial_cluster.sh | 5 ---- examples/local/203_vertical_split.sh | 4 --- examples/local/205_vertical_migrate_master.sh | 3 --- examples/local/206_clean_commerce.sh | 3 --- examples/local/302_new_shards.sh | 5 ---- examples/local/304_migrate_replicas.sh | 1 - examples/local/305_migrate_master.sh | 2 -- examples/local/306_down_shard_0.sh | 4 +-- examples/local/307_delete_shard_0.sh | 25 +++++++++++++++++++ examples/local/401_teardown.sh | 1 - examples/local/lmysql.sh | 19 ++++++++++++++ 11 files changed, 46 insertions(+), 26 deletions(-) mode change 100755 => 100644 examples/local/306_down_shard_0.sh create mode 100755 examples/local/307_delete_shard_0.sh create mode 100755 examples/local/lmysql.sh diff --git a/examples/local/101_initial_cluster.sh b/examples/local/101_initial_cluster.sh index 1c30fdff73f..921afde70fb 100755 --- a/examples/local/101_initial_cluster.sh +++ b/examples/local/101_initial_cluster.sh @@ -46,10 +46,5 @@ sleep 15 # start vtgate CELL=zone1 $script_root/vtgate-up.sh -sleep 5 - -# insert rows into the tables. we are actually connecting to vtgate here, it will route the sql correctly -mysql -h 127.0.0.1 -P 15306 -u mysql_user < $script_root/../common/insert_commerce_data.sql -mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_commerce_data.sql disown -a diff --git a/examples/local/203_vertical_split.sh b/examples/local/203_vertical_split.sh index 21afb3798c2..7a88c0d4860 100755 --- a/examples/local/203_vertical_split.sh +++ b/examples/local/203_vertical_split.sh @@ -30,8 +30,4 @@ $VTROOT/bin/vtworker \ -use_v3_resharding_mode \ VerticalSplitClone -min_healthy_rdonly_tablets=1 -tables=customer,corder customer/0 -sleep 2 -# check that customer and order data has been copied to customer keyspace -mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_customer0_data.sql - disown -a diff --git a/examples/local/205_vertical_migrate_master.sh b/examples/local/205_vertical_migrate_master.sh index d2024c05739..560564e5099 100755 --- a/examples/local/205_vertical_migrate_master.sh +++ b/examples/local/205_vertical_migrate_master.sh @@ -23,7 +23,4 @@ script_root=`dirname "${BASH_SOURCE}"` ./lvtctl.sh MigrateServedFrom customer/0 master -# customer and order data cannot be accessed from commerce keyspace -mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_commerce_data.sql - disown -a diff --git a/examples/local/206_clean_commerce.sh b/examples/local/206_clean_commerce.sh index 2b410a5d12e..eeef5d2c04f 100755 --- a/examples/local/206_clean_commerce.sh +++ b/examples/local/206_clean_commerce.sh @@ -26,7 +26,4 @@ script_root=`dirname "${BASH_SOURCE}"` ./lvtctl.sh SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 replica ./lvtctl.sh SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 master -# customer and order tables have now been deleted from commerce keyspace -mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_commerce_data.sql - disown -a diff --git a/examples/local/302_new_shards.sh b/examples/local/302_new_shards.sh index c6909d04d6d..30c1cb6b507 100755 --- a/examples/local/302_new_shards.sh +++ b/examples/local/302_new_shards.sh @@ -29,9 +29,4 @@ sleep 15 ./lvtctl.sh CopySchemaShard customer/0 customer/-80 ./lvtctl.sh CopySchemaShard customer/0 customer/80- -sleep 15 -# new tables are empty -mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_customer-80_data.sql -mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_customer80-_data.sql - disown -a diff --git a/examples/local/304_migrate_replicas.sh b/examples/local/304_migrate_replicas.sh index 9e5d381e515..e4c154dbea9 100755 --- a/examples/local/304_migrate_replicas.sh +++ b/examples/local/304_migrate_replicas.sh @@ -23,5 +23,4 @@ script_root=`dirname "${BASH_SOURCE}"` ./lvtctl.sh MigrateServedTypes customer/0 rdonly ./lvtctl.sh MigrateServedTypes customer/0 replica - disown -a diff --git a/examples/local/305_migrate_master.sh b/examples/local/305_migrate_master.sh index 5f989708f9f..0e52217d752 100755 --- a/examples/local/305_migrate_master.sh +++ b/examples/local/305_migrate_master.sh @@ -23,7 +23,5 @@ script_root=`dirname "${BASH_SOURCE}"` ./lvtctl.sh MigrateServedTypes customer/0 master # data has been copied over to shards, and databases for the new shards are now available -mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_customer-80_data.sql -mysql -h 127.0.0.1 -P 15306 -u mysql_user --table < $script_root/../common/select_customer80-_data.sql disown -a diff --git a/examples/local/306_down_shard_0.sh b/examples/local/306_down_shard_0.sh old mode 100755 new mode 100644 index 5d32383dea4..1fb25a8c8d5 --- a/examples/local/306_down_shard_0.sh +++ b/examples/local/306_down_shard_0.sh @@ -14,12 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this script deletes the old shard 0 which has been replaced by 2 shards +# this script brings down the tablets for customer/0 keyspace set -e script_root=`dirname "${BASH_SOURCE}"` -./lvtctl.sh DeleteShard -recursive customer/0 +CELL=zone1 UID_BASE=100 $script_root/vttablet-down.sh disown -a diff --git a/examples/local/307_delete_shard_0.sh b/examples/local/307_delete_shard_0.sh new file mode 100755 index 00000000000..5d32383dea4 --- /dev/null +++ b/examples/local/307_delete_shard_0.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2018 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script deletes the old shard 0 which has been replaced by 2 shards + +set -e + +script_root=`dirname "${BASH_SOURCE}"` + +./lvtctl.sh DeleteShard -recursive customer/0 + +disown -a diff --git a/examples/local/401_teardown.sh b/examples/local/401_teardown.sh index d0663ac7da4..ddc34f3f9c1 100755 --- a/examples/local/401_teardown.sh +++ b/examples/local/401_teardown.sh @@ -24,7 +24,6 @@ set -e script_root=`dirname "${BASH_SOURCE}"` ./vtgate-down.sh -CELL=zone1 UID_BASE=100 $script_root/vttablet-down.sh CELL=zone1 UID_BASE=200 $script_root/vttablet-down.sh CELL=zone1 UID_BASE=300 $script_root/vttablet-down.sh CELL=zone1 UID_BASE=400 $script_root/vttablet-down.sh diff --git a/examples/local/lmysql.sh b/examples/local/lmysql.sh new file mode 100755 index 00000000000..1382e517b8d --- /dev/null +++ b/examples/local/lmysql.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright 2018 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a convenience script to run mysql client against the local vtgate. + +mysql -h 127.0.0.1 -P 15306 -u mysql_user From 0ccc3b9ffff087b894886f60d2af0481b466866f Mon Sep 17 00:00:00 2001 From: deepthi Date: Fri, 7 Dec 2018 11:20:37 -0800 Subject: [PATCH 6/7] codeclimate fix Signed-off-by: deepthi --- examples/local/307_delete_shard_0.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/local/307_delete_shard_0.sh b/examples/local/307_delete_shard_0.sh index 5d32383dea4..284e6aba3ab 100755 --- a/examples/local/307_delete_shard_0.sh +++ b/examples/local/307_delete_shard_0.sh @@ -18,8 +18,6 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` - ./lvtctl.sh DeleteShard -recursive customer/0 disown -a From d97a1942d5997f0f3d0bb03642c6b9ce6c0167c7 Mon Sep 17 00:00:00 2001 From: deepthi Date: Fri, 7 Dec 2018 12:20:23 -0800 Subject: [PATCH 7/7] shellcheck fixes Signed-off-by: deepthi --- examples/local/101_initial_cluster.sh | 13 +++++++------ examples/local/202_customer_tablets.sh | 5 +++-- examples/local/203_vertical_split.sh | 13 +++++++++---- examples/local/204_vertical_migrate_replicas.sh | 2 -- examples/local/205_vertical_migrate_master.sh | 2 -- examples/local/206_clean_commerce.sh | 2 -- examples/local/301_customer_sharded.sh | 2 -- examples/local/302_new_shards.sh | 7 ++++--- examples/local/303_horizontal_split.sh | 13 +++++++++---- examples/local/304_migrate_replicas.sh | 2 -- examples/local/305_migrate_master.sh | 3 --- examples/local/306_down_shard_0.sh | 5 +++-- examples/local/401_teardown.sh | 13 +++++++------ 13 files changed, 42 insertions(+), 40 deletions(-) mode change 100644 => 100755 examples/local/306_down_shard_0.sh diff --git a/examples/local/101_initial_cluster.sh b/examples/local/101_initial_cluster.sh index 921afde70fb..a3f9793f290 100755 --- a/examples/local/101_initial_cluster.sh +++ b/examples/local/101_initial_cluster.sh @@ -19,20 +19,21 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` +# shellcheck disable=SC2128 +script_root=$(dirname "${BASH_SOURCE}") # start topo server if [ "${TOPO}" = "etcd2" ]; then - CELL=zone1 $script_root/etcd-up.sh + CELL=zone1 "$script_root/etcd-up.sh" else - CELL=zone1 $script_root/zk-up.sh + CELL=zone1 "$script_root/zk-up.sh" fi # start vtctld -CELL=zone1 $script_root/vtctld-up.sh +CELL=zone1 "$script_root/vtctld-up.sh" # start vttablets for keyspace commerce -CELL=zone1 KEYSPACE=commerce UID_BASE=100 $script_root/vttablet-up.sh +CELL=zone1 KEYSPACE=commerce UID_BASE=100 "$script_root/vttablet-up.sh" sleep 15 # set one of the replicas to master @@ -45,6 +46,6 @@ sleep 15 ./lvtctl.sh ApplyVSchema -vschema_file vschema_commerce_initial.json commerce # start vtgate -CELL=zone1 $script_root/vtgate-up.sh +CELL=zone1 "$script_root/vtgate-up.sh" disown -a diff --git a/examples/local/202_customer_tablets.sh b/examples/local/202_customer_tablets.sh index 75deccbacb6..05cc4ff9e6e 100755 --- a/examples/local/202_customer_tablets.sh +++ b/examples/local/202_customer_tablets.sh @@ -20,9 +20,10 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` +# shellcheck disable=SC2128 +script_root=$(dirname "${BASH_SOURCE}") -CELL=zone1 KEYSPACE=customer UID_BASE=200 $script_root/vttablet-up.sh +CELL=zone1 KEYSPACE=customer UID_BASE=200 "$script_root/vttablet-up.sh" sleep 15 ./lvtctl.sh InitShardMaster -force customer/0 zone1-200 ./lvtctl.sh CopySchemaShard -tables customer,corder commerce/0 customer/0 diff --git a/examples/local/203_vertical_split.sh b/examples/local/203_vertical_split.sh index 7a88c0d4860..d5a67d6c8f7 100755 --- a/examples/local/203_vertical_split.sh +++ b/examples/local/203_vertical_split.sh @@ -19,13 +19,18 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` -source $script_root/env.sh +# shellcheck disable=SC2128 +script_root=$(dirname "${BASH_SOURCE}") -$VTROOT/bin/vtworker \ +# shellcheck source=./env.sh +# shellcheck disable=SC1091 +source "$script_root/env.sh" + +# shellcheck disable=SC2086 +"$VTROOT"/bin/vtworker \ $TOPOLOGY_FLAGS \ -cell zone1 \ - -log_dir $VTDATAROOT/tmp \ + -log_dir "$VTDATAROOT"/tmp \ -alsologtostderr \ -use_v3_resharding_mode \ VerticalSplitClone -min_healthy_rdonly_tablets=1 -tables=customer,corder customer/0 diff --git a/examples/local/204_vertical_migrate_replicas.sh b/examples/local/204_vertical_migrate_replicas.sh index 567dca6f508..3bb6c6dccbf 100755 --- a/examples/local/204_vertical_migrate_replicas.sh +++ b/examples/local/204_vertical_migrate_replicas.sh @@ -19,8 +19,6 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` - ./lvtctl.sh MigrateServedFrom customer/0 rdonly ./lvtctl.sh MigrateServedFrom customer/0 replica diff --git a/examples/local/205_vertical_migrate_master.sh b/examples/local/205_vertical_migrate_master.sh index 560564e5099..1b42b0cab0d 100755 --- a/examples/local/205_vertical_migrate_master.sh +++ b/examples/local/205_vertical_migrate_master.sh @@ -19,8 +19,6 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` - ./lvtctl.sh MigrateServedFrom customer/0 master disown -a diff --git a/examples/local/206_clean_commerce.sh b/examples/local/206_clean_commerce.sh index eeef5d2c04f..2aa66b2f91d 100755 --- a/examples/local/206_clean_commerce.sh +++ b/examples/local/206_clean_commerce.sh @@ -19,8 +19,6 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` - ./lvtctl.sh ApplySchema -sql-file drop_commerce_tables.sql commerce ./lvtctl.sh SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 rdonly ./lvtctl.sh SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 replica diff --git a/examples/local/301_customer_sharded.sh b/examples/local/301_customer_sharded.sh index c7f766810d6..2932f5d8776 100755 --- a/examples/local/301_customer_sharded.sh +++ b/examples/local/301_customer_sharded.sh @@ -22,8 +22,6 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` - ./lvtctl.sh ApplySchema -sql-file create_commerce_seq.sql commerce ./lvtctl.sh ApplyVSchema -vschema_file vschema_commerce_seq.json commerce ./lvtctl.sh ApplySchema -sql-file create_customer_sharded.sql customer diff --git a/examples/local/302_new_shards.sh b/examples/local/302_new_shards.sh index 30c1cb6b507..e70c211b847 100755 --- a/examples/local/302_new_shards.sh +++ b/examples/local/302_new_shards.sh @@ -19,10 +19,11 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` +# shellcheck disable=SC2128 +script_root=$(dirname "${BASH_SOURCE}") -SHARD=-80 CELL=zone1 KEYSPACE=customer UID_BASE=300 $script_root/vttablet-up.sh -SHARD=80- CELL=zone1 KEYSPACE=customer UID_BASE=400 $script_root/vttablet-up.sh +SHARD=-80 CELL=zone1 KEYSPACE=customer UID_BASE=300 "$script_root/vttablet-up.sh" +SHARD=80- CELL=zone1 KEYSPACE=customer UID_BASE=400 "$script_root/vttablet-up.sh" sleep 15 ./lvtctl.sh InitShardMaster -force customer/-80 zone1-300 ./lvtctl.sh InitShardMaster -force customer/80- zone1-400 diff --git a/examples/local/303_horizontal_split.sh b/examples/local/303_horizontal_split.sh index 6851f6c378d..9fb583d9dc1 100755 --- a/examples/local/303_horizontal_split.sh +++ b/examples/local/303_horizontal_split.sh @@ -19,13 +19,18 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` -source $script_root/env.sh +# shellcheck disable=SC2128 +script_root=$(dirname "${BASH_SOURCE}") -$VTROOT/bin/vtworker \ +# shellcheck source=./env.sh +# shellcheck disable=SC1091 +source "${script_root}/env.sh" + +# shellcheck disable=SC2086 +"$VTROOT"/bin/vtworker \ $TOPOLOGY_FLAGS \ -cell zone1 \ - -log_dir $VTDATAROOT/tmp \ + -log_dir "$VTDATAROOT"/tmp \ -alsologtostderr \ -use_v3_resharding_mode \ SplitClone -min_healthy_rdonly_tablets=1 customer/0 diff --git a/examples/local/304_migrate_replicas.sh b/examples/local/304_migrate_replicas.sh index e4c154dbea9..39c86fe60e2 100755 --- a/examples/local/304_migrate_replicas.sh +++ b/examples/local/304_migrate_replicas.sh @@ -18,8 +18,6 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` - ./lvtctl.sh MigrateServedTypes customer/0 rdonly ./lvtctl.sh MigrateServedTypes customer/0 replica diff --git a/examples/local/305_migrate_master.sh b/examples/local/305_migrate_master.sh index 0e52217d752..f5f1170ebc0 100755 --- a/examples/local/305_migrate_master.sh +++ b/examples/local/305_migrate_master.sh @@ -18,10 +18,7 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` - ./lvtctl.sh MigrateServedTypes customer/0 master - # data has been copied over to shards, and databases for the new shards are now available disown -a diff --git a/examples/local/306_down_shard_0.sh b/examples/local/306_down_shard_0.sh old mode 100644 new mode 100755 index 1fb25a8c8d5..530746846d0 --- a/examples/local/306_down_shard_0.sh +++ b/examples/local/306_down_shard_0.sh @@ -18,8 +18,9 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` +# shellcheck disable=SC2128 +script_root=$(dirname "${BASH_SOURCE}") -CELL=zone1 UID_BASE=100 $script_root/vttablet-down.sh +CELL=zone1 UID_BASE=100 "$script_root/vttablet-down.sh" disown -a diff --git a/examples/local/401_teardown.sh b/examples/local/401_teardown.sh index ddc34f3f9c1..f314b6a6073 100755 --- a/examples/local/401_teardown.sh +++ b/examples/local/401_teardown.sh @@ -21,18 +21,19 @@ set -e -script_root=`dirname "${BASH_SOURCE}"` +# shellcheck disable=SC2128 +script_root=$(dirname "${BASH_SOURCE}") ./vtgate-down.sh -CELL=zone1 UID_BASE=200 $script_root/vttablet-down.sh -CELL=zone1 UID_BASE=300 $script_root/vttablet-down.sh -CELL=zone1 UID_BASE=400 $script_root/vttablet-down.sh +CELL=zone1 UID_BASE=200 "$script_root/vttablet-down.sh" +CELL=zone1 UID_BASE=300 "$script_root/vttablet-down.sh" +CELL=zone1 UID_BASE=400 "$script_root/vttablet-down.sh" ./vtctld-down.sh if [ "${TOPO}" = "etcd2" ]; then - CELL=zone1 $script_root/etcd-down.sh + CELL=zone1 "$script_root/etcd-down.sh" else - CELL=zone1 $script_root/zk-down.sh + CELL=zone1 "$script_root/zk-down.sh" fi disown -a