Sync with upstream vitess; 2018-12-14

Vagrantfile

@@ -39,10 +39,13 @@ Vagrant.configure("2") do |config|
   # Demo Appp
   config.vm.network "forwarded_port", guest: 8000, host: 8000 # http
 
-  # If possible, use nfs, this gives a good boost to IO operations in the VM.
-  # if you run into with nfs, just remove this from the synced folder
-
-  config.vm.synced_folder ".", "/vagrant/src/vitess.io/vitess", type: "nfs"
+  # N.B. It's possible to use NFS to help speed up IO operations in the VM but
+  # some OSX users have reported issues running govendor with it enabled.
+  # Additional details in https://github.com/vitessio/vitess/issues/4396
+  #
+  # To enable, use:
+  #   config.vm.synced_folder ".", "/vagrant/src/vitess.io/vitess", type: "nfs"
+  config.vm.synced_folder ".", "/vagrant/src/vitess.io/vitess"
 
   config.vm.provider :virtualbox do |vb|
     vb.name = "vitess"

config/mycnf/master_mysql56.cnf

@@ -4,8 +4,6 @@ gtid_mode = ON
 log_bin
 log_slave_updates
 enforce_gtid_consistency
-secure_file_priv = NULL
-innodb_support_xa = 0
 
 # Crash-safe replication settings.
 master_info_repository = TABLE

doc/vtctlReference.md

@@ -1119,8 +1119,11 @@ Applies the VTGate routing schema to the provided keyspace. Shows the result aft
 | :-------- | :--------- | :--------- |
 | cells | string | If specified, limits the rebuild to the cells, after upload. Ignored if skipRebuild is set. |
 | skip_rebuild | Boolean | If set, do no rebuild the SrvSchema objects. |
+| dry-run | Boolean | Shows the proposed change without executing it |
 | vschema | string | Identifies the VTGate routing schema |
 | vschema_file | string | Identifies the VTGate routing schema file |
+| sql | string | Identifies a VSchema DDL SQL statement |
+| sql_file | string | Identifies a VSchema DDL SQL statement |
 
 
 #### Arguments

examples/common/insert_commerce_data.sql

@@ -0,0 +1,12 @@
+insert into customer(email) values('alice@domain.com');
+insert into customer(email) values('bob@domain.com');
+insert into customer(email) values('charlie@domain.com');
+insert into customer(email) values('dan@domain.com');
+insert into customer(email) values('eve@domain.com');
+insert into product(sku, description, price) values('SKU-1001', 'Monitor', 100);
+insert into product(sku, description, price) values('SKU-1002', 'Keyboard', 30);
+insert into corder(customer_id, sku, price) values(1, 'SKU-1001', 100);
+insert into corder(customer_id, sku, price) values(2, 'SKU-1002', 30);
+insert into corder(customer_id, sku, price) values(3, 'SKU-1002', 30);
+insert into corder(customer_id, sku, price) values(4, 'SKU-1002', 30);
+insert into corder(customer_id, sku, price) values(5, 'SKU-1002', 30);

examples/common/select_commerce_data.sql

@@ -0,0 +1,8 @@
+\! echo 'Using commerce/0'
+use commerce/0;
+\! echo 'Customer'
+select * from customer;
+\! echo 'Product'
+select * from product;
+\! echo 'COrder'
+select * from corder;

examples/common/select_customer-80_data.sql

@@ -0,0 +1,6 @@
+\! echo 'Using customer/-80'
+use customer/-80;
+\! echo 'Customer'
+select * from customer;
+\! echo 'COrder'
+select * from corder;

examples/common/select_customer0_data.sql

@@ -0,0 +1,6 @@
+\! echo 'Using customer/0'
+use customer/0;
+\! echo 'Customer'
+select * from customer;
+\! echo 'COrder'
+select * from corder;

examples/common/select_customer80-_data.sql

@@ -0,0 +1,6 @@
+\! echo 'Using customer/80-'
+use customer/80-;
+\! echo 'Customer'
+select * from customer;
+\! echo 'COrder'
+select * from corder;

examples/local/101_initial_cluster.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Copyright 2018 The Vitess Authors.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script brings up zookeeper and all the vitess components
+# required for a single shard deployment.
+
+set -e
+
+# shellcheck disable=SC2128
+script_root=$(dirname "${BASH_SOURCE}")
+
+# start topo server
+if [ "${TOPO}" = "etcd2" ]; then
+    CELL=zone1 "$script_root/etcd-up.sh"
+else
+    CELL=zone1 "$script_root/zk-up.sh"
+fi
+
+# start vtctld
+CELL=zone1 "$script_root/vtctld-up.sh"
+
+# start vttablets for keyspace commerce
+CELL=zone1 KEYSPACE=commerce UID_BASE=100 "$script_root/vttablet-up.sh"
+sleep 15
+
+# set one of the replicas to master
+./lvtctl.sh InitShardMaster -force commerce/0 zone1-100
+
+# create the schema
+./lvtctl.sh ApplySchema -sql-file create_commerce_schema.sql commerce
+
+# create the vschema
+./lvtctl.sh ApplyVSchema -vschema_file vschema_commerce_initial.json commerce
+
+# start vtgate
+CELL=zone1 "$script_root/vtgate-up.sh"
+
+disown -a

examples/local/sharded-vttablet-down.sh → examples/local/201_customer_keyspace.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2017 Google Inc.
+# Copyright 2018 The Vitess Authors.
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,11 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This is an example script that stops the mysqld and vttablet instances
-# created by sharded-vttablet-up.sh
+# this script creates a new keyspace in preparation for vertical resharding
 
-script_root=`dirname "${BASH_SOURCE}"`
+set -e
 
-UID_BASE=200 $script_root/vttablet-down.sh "$@"
-UID_BASE=300 $script_root/vttablet-down.sh "$@"
+./lvtctl.sh CreateKeyspace -served_from='master:commerce,replica:commerce,rdonly:commerce' customer
 
+disown -a

examples/local/202_customer_tablets.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2018 The Vitess Authors.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script creates the tablets and initializes them for vertical
+# resharding it also splits the vschema between the two keyspaces
+# old (commerce) and new (customer)
+
+set -e
+
+# shellcheck disable=SC2128
+script_root=$(dirname "${BASH_SOURCE}")
+
+CELL=zone1 KEYSPACE=customer UID_BASE=200 "$script_root/vttablet-up.sh"
+sleep 15
+./lvtctl.sh InitShardMaster -force customer/0 zone1-200
+./lvtctl.sh CopySchemaShard -tables customer,corder commerce/0 customer/0
+./lvtctl.sh ApplyVSchema -vschema_file vschema_commerce_vsplit.json commerce
+./lvtctl.sh ApplyVSchema -vschema_file vschema_customer_vsplit.json customer
+
+disown -a

examples/local/203_vertical_split.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright 2018 The Vitess Authors.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script copies over all the data from commerce keyspace to
+# customer keyspace for the customer and corder tables
+
+set -e
+
+# shellcheck disable=SC2128
+script_root=$(dirname "${BASH_SOURCE}")
+
+# shellcheck source=./env.sh
+# shellcheck disable=SC1091
+source "$script_root/env.sh"
+
+# shellcheck disable=SC2086
+"$VTROOT"/bin/vtworker \
+    $TOPOLOGY_FLAGS \
+    -cell zone1 \
+    -log_dir "$VTDATAROOT"/tmp \
+    -alsologtostderr \
+    -use_v3_resharding_mode \
+    VerticalSplitClone -min_healthy_rdonly_tablets=1 -tables=customer,corder customer/0
+
+disown -a

examples/local/sharded-vttablet-up.sh → examples/local/204_vertical_migrate_replicas.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2017 Google Inc.
+# Copyright 2018 The Vitess Authors.
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,16 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This is an example script that creates a sharded vttablet deployment.
+# this script migrates traffic for the new customer keyspace to the new
+# tablets of types rdonly and replica
 
 set -e
 
-script_root=`dirname "${BASH_SOURCE}"`
-
-# Shard -80 contains all entries whose keyspace ID has a first byte < 0x80.
-# See: http://vitess.io/overview/concepts/#keyspace-id
-SHARD=-80 UID_BASE=200 $script_root/vttablet-up.sh "$@"
-
-# Shard 80- contains all entries whose keyspace ID has a first byte >= 0x80.
-SHARD=80- UID_BASE=300 $script_root/vttablet-up.sh "$@"
+./lvtctl.sh MigrateServedFrom customer/0 rdonly
+./lvtctl.sh MigrateServedFrom customer/0 replica
 
+disown -a

examples/local/205_vertical_migrate_master.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright 2018 The Vitess Authors.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script migrates master traffic for the customer keyspace to the
+# new master tablet
+
+set -e
+
+./lvtctl.sh MigrateServedFrom customer/0 master
+
+disown -a

examples/local/206_clean_commerce.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright 2018 The Vitess Authors.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script removes the customer and corder tables from the commerce
+# keyspace
+
+set -e
+
+./lvtctl.sh ApplySchema -sql-file drop_commerce_tables.sql commerce
+./lvtctl.sh SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 rdonly
+./lvtctl.sh SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 replica
+./lvtctl.sh SetShardTabletControl -blacklisted_tables=customer,corder -remove commerce/0 master
+
+disown -a

examples/local/301_customer_sharded.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright 2018 The Vitess Authors.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script creates vitess sequences for the auto_increment fields
+# and alters the fields to no longer be auto_increment in preparation
+# for horizontal sharding
+# it also changes the customer vschema from unsharded to sharded and
+# sets up the necessary vindexes
+
+set -e
+
+./lvtctl.sh ApplySchema -sql-file create_commerce_seq.sql commerce
+./lvtctl.sh ApplyVSchema -vschema_file vschema_commerce_seq.json commerce
+./lvtctl.sh ApplySchema -sql-file create_customer_sharded.sql customer
+./lvtctl.sh ApplyVSchema -vschema_file vschema_customer_sharded.json customer
+
+disown -a

examples/local/302_new_shards.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2018 The Vitess Authors.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script brings up new tablets for the two new shards that we will
+# be creating in the customer keyspace and copies the schema 
+
+set -e
+
+# shellcheck disable=SC2128
+script_root=$(dirname "${BASH_SOURCE}")
+
+SHARD=-80 CELL=zone1 KEYSPACE=customer UID_BASE=300 "$script_root/vttablet-up.sh"
+SHARD=80- CELL=zone1 KEYSPACE=customer UID_BASE=400 "$script_root/vttablet-up.sh"
+sleep 15
+./lvtctl.sh InitShardMaster -force customer/-80 zone1-300
+./lvtctl.sh InitShardMaster -force customer/80- zone1-400
+./lvtctl.sh CopySchemaShard customer/0 customer/-80
+./lvtctl.sh CopySchemaShard customer/0 customer/80-
+
+disown -a