Slack sync upstream 2018.08.09r1

.gitignore

@@ -70,3 +70,6 @@ releases
 /web/vtctld2/bower.json~
 /web/vtctld2/public/bower_components/
 
+
+# Vagrant
+.vagrant

Makefile

@@ -135,7 +135,6 @@ PROTO_SRCS = $(wildcard proto/*.proto)
 PROTO_SRC_NAMES = $(basename $(notdir $(PROTO_SRCS)))
 PROTO_PY_OUTS = $(foreach name, $(PROTO_SRC_NAMES), py/vtproto/$(name)_pb2.py)
 PROTO_GO_OUTS = $(foreach name, $(PROTO_SRC_NAMES), go/vt/proto/$(name)/$(name).pb.go)
-PROTO_GO_TEMPS = $(foreach name, $(PROTO_SRC_NAMES), go/vt/.proto.tmp/$(name).pb.go)
 
 # This rule rebuilds all the go and python files from the proto definitions for gRPC.
 proto: proto_banner $(PROTO_GO_OUTS) $(PROTO_PY_OUTS)
@@ -152,19 +151,11 @@ endif
 $(PROTO_PY_OUTS): py/vtproto/%_pb2.py: proto/%.proto
 	$(PROTOC_COMMAND) -Iproto $< --python_out=py/vtproto --grpc_python_out=py/vtproto
 
-$(PROTO_GO_OUTS): $(PROTO_GO_TEMPS)
+$(PROTO_GO_OUTS): install_protoc-gen-go proto/*.proto
 	for name in $(PROTO_SRC_NAMES); do \
-		mkdir -p go/vt/proto/$${name}; \
-		cp -a go/vt/.proto.tmp/$${name}.pb.go go/vt/proto/$${name}/$${name}.pb.go; \
+		cd $(VTROOT)/src && PATH=$(VTROOT)/bin:$(PATH) $(VTROOT)/bin/protoc --go_out=plugins=grpc:. -Ivitess.io/vitess/proto vitess.io/vitess/proto/$${name}.proto; \
 	done
 
-$(PROTO_GO_TEMPS): install_protoc-gen-go
-
-$(PROTO_GO_TEMPS): go/vt/.proto.tmp/%.pb.go: proto/%.proto
-	mkdir -p go/vt/.proto.tmp
-	$(PROTOC_COMMAND) -Iproto $< --plugin=grpc=protoc-gen-go --go_out=plugins=grpc:go/vt/.proto.tmp
-	sed -i -e 's,import \([a-z0-9_]*\) ".",import \1 "vitess.io/vitess/go/vt/proto/\1",g' $@
-
 # Helper targets for building Docker images.
 # Please read docker/README.md to understand the different available images.
 

Vagrantfile

@@ -0,0 +1,56 @@
+ENV["LC_ALL"] = "en_US.UTF-8"
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/xenial64"
+  config.vm.hostname = 'vitess'
+
+  config.vm.network "private_network", type: "dhcp"
+
+  # vtctld
+  config.vm.network "forwarded_port", guest: 8000, host: 8000 # http
+  config.vm.network "forwarded_port", guest: 15000, host: 15000 # http
+  config.vm.network "forwarded_port", guest: 15999, host: 15999 # grpc
+
+  # vtgate
+  config.vm.network "forwarded_port", guest: 15001, host: 15001 # http
+  config.vm.network "forwarded_port", guest: 15991, host: 15991 # grpc
+  config.vm.network "forwarded_port", guest: 15306, host: 15306 # mysql
+
+  # vttablet 1
+  config.vm.network "forwarded_port", guest: 15100, host: 15100 # http
+  config.vm.network "forwarded_port", guest: 16100, host: 16100 # grpc
+
+  # vttablet 2
+  config.vm.network "forwarded_port", guest: 15101, host: 15101 # http
+  config.vm.network "forwarded_port", guest: 16101, host: 16101 # grpc
+
+  # vttablet 3
+  config.vm.network "forwarded_port", guest: 15102, host: 15102 # http
+  config.vm.network "forwarded_port", guest: 16102, host: 16102 # grpc
+
+  # vttablet 4
+  config.vm.network "forwarded_port", guest: 15103, host: 15103 # http
+  config.vm.network "forwarded_port", guest: 16103, host: 16103 # grpc
+
+  # vttablet 5
+  config.vm.network "forwarded_port", guest: 15104, host: 15104 # http
+  config.vm.network "forwarded_port", guest: 16104, host: 16104 # grpc
+
+  # Demo Appp
+  config.vm.network "forwarded_port", guest: 8000, host: 8000 # http
+
+  # If possible, use nfs, this gives a good boost to IO operations in the VM.
+  # if you run into with nfs, just remove this from the synced folder
+
+  config.vm.synced_folder ".", "/vagrant/src/vitess.io/vitess", type: "nfs"
+
+  config.vm.provider :virtualbox do |vb|
+    vb.name = "vitess"
+    vb.customize ["modifyvm", :id, "--ioapic", "on"]
+    vb.customize ["modifyvm", :id, "--cpuexecutioncap", "85"]
+    vb.customize [ "modifyvm", :id, "--uartmode1", "disconnected" ]
+    vb.memory = 12888
+    vb.cpus = 4
+  end
+  config.vm.provision "shell", path: "./vagrant-scripts/bootstrap_vm.sh"
+end

bootstrap.sh

@@ -139,6 +139,24 @@ function install_grpc() {
 install_dep "gRPC" "1.10.0" "$VTROOT/dist/grpc" install_grpc
 
 
+# Install protoc.
+function install_protoc() {
+  local version="$1"
+  local dist="$2"
+
+  case $(uname) in
+    Linux)  local platform=linux;;
+    Darwin) local platform=osx;;
+  esac
+
+  wget "https://github.com/google/protobuf/releases/download/v$version/protoc-$version-$platform-x86_64.zip"
+  unzip "protoc-$version-$platform-x86_64.zip"
+  ln -snf "$dist/bin/protoc" "$VTROOT/bin/protoc"
+}
+protoc_ver=3.5.1
+install_dep "protoc" "$protoc_ver" "$VTROOT/dist/vt-protoc-$protoc_ver" install_protoc
+
+
 # Install Zookeeper.
 function install_zookeeper() {
   local version="$1"
@@ -233,7 +251,7 @@ function install_chromedriver() {
   unzip -o -q chromedriver_linux64.zip -d "$dist"
   rm chromedriver_linux64.zip
 }
-install_dep "chromedriver" "2.25" "$VTROOT/dist/chromedriver" install_chromedriver
+install_dep "chromedriver" "2.40" "$VTROOT/dist/chromedriver" install_chromedriver
 
 
 #

config/mycnf/master_mysql56.cnf

@@ -5,8 +5,11 @@ log_bin
 log_slave_updates
 enforce_gtid_consistency
 
-# Ignore relay logs on disk at startup.
-relay_log_recovery
+# Crash-safe replication settings.
+master_info_repository = TABLE
+relay_log_info_repository = TABLE
+relay_log_purge = 1
+relay_log_recovery = 1
 
 # Native AIO tends to run into aio-max-nr limit during test startup.
 innodb_use_native_aio = 0

dev.env

@@ -50,7 +50,7 @@ IFS="
 "
 # Note that the escaped ( ) around the -or expression are important.
 # Otherwise, the -print would match *all* files.
-for p in $(find "$VTROOT/dist" \( -name site-packages -or -name dist-packages \) -print); do
+for pypath in $(find "$VTROOT/dist" \( -name site-packages -or -name dist-packages \) -print); do
   PYTHONPATH=$(prepend_path "$PYTHONPATH" "$pypath")
 done
 IFS="$BACKUP_IFS"

doc/SeparatingVttabletMysql.md

@@ -20,9 +20,9 @@ The following adjustments need to be made to VTTablet command line parameters:
 
 * Do not use `-mycnf_socket_file`. There is no local MySQL unix socket file.
 
-* Specify the host and port of the MySQL daemon for the `-db-config-XXX-host`
-  and `-db-config-XXX-port` command line parameters. Do not specify
-  `-db-config-XXX-unixsocket` parameters.
+* Specify the host and port of the MySQL daemon for the `-db\_host`
+  and `-db\_port` command line parameters. Do not specify
+  `-db\_socket` parameters.
 
 * Disable restores / backups, by not passing any backup command line
   parameters. Specifically, `-restore_from_backup` and

doc/ServerConfiguration.md

@@ -256,36 +256,45 @@ the binlogs in `/mnt/bin-logs`:
 
 VTTablet requires multiple user credentials to perform its tasks. Since it's required to run on the same machine as MySQL, it’s most beneficial to use the more efficient unix socket connections.
 
+**connection** parameters
+
+* **db\_socket**: The unix socket to connect on. If this is specifed, host and port will not be used.
+* **db\_host**: The host name for the tcp connection.
+* **db\_port**: The tcp port to be used with the db\_host.
+* **db\_charset**: Character set. Only utf8 or latin1 based character sets are supported.
+* **db\_flags**: Flag values as defined by MySQL.
+* **db\ssl\_ca, db\_ssl\_ca\_path, db\_ssl\_cert, db\_ssl\_key**: SSL flags.
+
+
 **app** credentials are for serving app queries:
 
-* **db-config-app-unixsocket**: MySQL socket name to connect to.
-* **db-config-app-uname**: App username.
-* **db-config-app-pass**: Password for the app username. If you need a more secure way of managing and supplying passwords, VTTablet does allow you to plug into a "password server" that can securely supply and refresh usernames and passwords. Please contact the Vitess team for help if you’d like to write such a custom plugin.
-* **db-config-app-charset**: The only supported character set is utf8. Vitess still works with latin1, but it’s getting deprecated.
+* **db\_app\_user**: App username.
+* **db\_app\_password**: Password for the app username. If you need a more secure way of managing and supplying passwords, VTTablet does allow you to plug into a "password server" that can securely supply and refresh usernames and passwords. Please contact the Vitess team for help if you’d like to write such a custom plugin.
+* **db\_app\_use\_ssl**: Set this flag to false if you don't want to use SSL for this connection. This will allow you to turn off SSL for all users except for `repl`, which may have to be turned on for replication that goes over open networks.
+
+**appdebug** credentials are for the appdebug user:
+
+* **db\_appdebug\_user**
+* **db\_appdebug\_password**
+* **db\_appdebug\_use\_ssl**
 
 **dba** credentials will be used for housekeeping work like loading the schema or killing runaway queries:
 
-* **db-config-dba-unixsocket**
-* **db-config-dba-uname**
-* **db-config-dba-pass**
-* **db-config-dba-charset**
+* **db\_dba\_user**
+* **db\_dba\_password**
+* **db\_dba\_use\_ssl**
 
-**repl** credentials are for managing replication. Since repl connections can be used across machines, you can optionally turn on encryption:
+**repl** credentials are for managing replication.
 
-* **db-config-repl-uname**
-* **db-config-repl-pass**
-* **db-config-repl-charset**
-* **db-config-repl-flags**: If you want to enable SSL, this must be set to 2048.
-* **db-config-repl-ssl-ca**
-* **db-config-repl-ssl-cert**
-* **db-config-repl-ssl-key**
+* **db\_repl\_user**
+* **db\_repl\_password**
+* **db\_repl\_use\_ssl**
 
 **filtered** credentials are for performing resharding:
 
-* **db-config-filtered-unixsocket**
-* **db-config-filtered-uname**
-* **db-config-filtered-pass**
-* **db-config-filtered-charset**
+* **db\_filtered\_user**
+* **db\_filtered\_password**
+* **db\_filtered\_use\_ssl**
 
 ### Monitoring
 

examples/compose/vttablet-up.sh

@@ -14,22 +14,6 @@ if [ "$uid" = "1" ]; then
     tablet_role='master'
 fi
 
-dbconfig_dba_flags="\
-    -db-config-dba-uname root \
-    -db-config-dba-charset utf8"
-
-dbconfig_flags="$dbconfig_dba_flags \
-    -db-config-app-uname root \
-    -db-config-app-charset utf8 \
-    -db-config-appdebug-uname root \
-    -db-config-appdebug-charset utf8 \
-    -db-config-allprivs-uname root \
-    -db-config-allprivs-charset utf8 \
-    -db-config-repl-uname root \
-    -db-config-repl-charset utf8 \
-    -db-config-filtered-uname root \
-    -db-config-filtered-charset utf8"
-
 init_db_sql_file="$VTROOT/init_db.sql"
 echo "GRANT ALL ON *.* TO 'root'@'%';" > $init_db_sql_file
 
@@ -52,7 +36,6 @@ fi
 $VTROOT/bin/mysqlctl \
   -log_dir $VTDATAROOT/tmp \
   -tablet_uid $uid \
-  $dbconfig_dba_flags \
   -mysql_port 3306 \
   $action &
 
@@ -78,5 +61,4 @@ exec $VTROOT/bin/vttablet \
   -grpc_port $GRPC_PORT \
   -service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \
   -pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \
-  -vtctld_addr http://vtctld:$WEB_PORT/ \
-  $dbconfig_flags
+  -vtctld_addr "http://vtctld:$WEB_PORT/"

examples/kubernetes/vttablet-pod-benchmarking-template.yaml

@@ -56,18 +56,6 @@ spec:
           -init_shard {{shard}}
           -init_tablet_type {{tablet_type}}
           -mysqlctl_socket $VTDATAROOT/mysqlctl.sock
-          -db-config-app-uname vt_app
-          -db-config-app-dbname vt_{{keyspace}}
-          -db-config-app-charset utf8
-          -db-config-dba-uname vt_dba
-          -db-config-dba-dbname vt_{{keyspace}}
-          -db-config-dba-charset utf8
-          -db-config-repl-uname vt_repl
-          -db-config-repl-dbname vt_{{keyspace}}
-          -db-config-repl-charset utf8
-          -db-config-filtered-uname vt_filtered
-          -db-config-filtered-dbname vt_{{keyspace}}
-          -db-config-filtered-charset utf8
           -queryserver-config-transaction-cap 300
           -queryserver-config-schema-reload-time 1
           -queryserver-config-pool-size 100
@@ -98,8 +86,6 @@ spec:
           -alsologtostderr
           -tablet_uid {{uid}}
           -socket_file $VTDATAROOT/mysqlctl.sock
-          -db-config-dba-uname vt_dba
-          -db-config-dba-charset utf8
           -init_db_sql_file $VTROOT/config/init_db.sql" vitess
       env:
         - name: EXTRA_MY_CNF

examples/kubernetes/vttablet-pod-template.yaml

@@ -64,18 +64,6 @@ spec:
           -init_tablet_type {{tablet_type}}
           -health_check_interval 5s
           -mysqlctl_socket $VTDATAROOT/mysqlctl.sock
-          -db-config-app-uname vt_app
-          -db-config-app-dbname vt_{{keyspace}}
-          -db-config-app-charset utf8
-          -db-config-dba-uname vt_dba
-          -db-config-dba-dbname vt_{{keyspace}}
-          -db-config-dba-charset utf8
-          -db-config-repl-uname vt_repl
-          -db-config-repl-dbname vt_{{keyspace}}
-          -db-config-repl-charset utf8
-          -db-config-filtered-uname vt_filtered
-          -db-config-filtered-dbname vt_{{keyspace}}
-          -db-config-filtered-charset utf8
           -enable_semi_sync
           -enable_replication_reporter
           -orc_api_url http://orchestrator/api
@@ -107,8 +95,6 @@ spec:
           -alsologtostderr
           -tablet_uid {{uid}}
           -socket_file $VTDATAROOT/mysqlctl.sock
-          -db-config-dba-uname vt_dba
-          -db-config-dba-charset utf8
           -init_db_sql_file $VTROOT/config/init_db.sql" vitess
       env:
         - name: EXTRA_MY_CNF

examples/local/vttablet-down.sh

@@ -41,7 +41,6 @@ for uid_index in $uids; do
 
   echo "Stopping MySQL for tablet $alias..."
   $VTROOT/bin/mysqlctl \
-    -db-config-dba-uname vt_dba \
     -tablet_uid $uid \
     shutdown &
 done

examples/local/vttablet-up.sh

@@ -36,25 +36,6 @@ fi
 script_root=`dirname "${BASH_SOURCE}"`
 source $script_root/env.sh
 
-dbconfig_dba_flags="\
-    -db-config-dba-uname vt_dba \
-    -db-config-dba-charset utf8"
-dbconfig_flags="$dbconfig_dba_flags \
-    -db-config-app-uname vt_app \
-    -db-config-app-dbname vt_$keyspace \
-    -db-config-app-charset utf8 \
-    -db-config-appdebug-uname vt_appdebug \
-    -db-config-appdebug-dbname vt_$keyspace \
-    -db-config-appdebug-charset utf8 \
-    -db-config-allprivs-uname vt_allprivs \
-    -db-config-allprivs-dbname vt_$keyspace \
-    -db-config-allprivs-charset utf8 \
-    -db-config-repl-uname vt_repl \
-    -db-config-repl-dbname vt_$keyspace \
-    -db-config-repl-charset utf8 \
-    -db-config-filtered-uname vt_filtered \
-    -db-config-filtered-dbname vt_$keyspace \
-    -db-config-filtered-charset utf8"
 init_db_sql_file="$VTROOT/config/init_db.sql"
 
 case "$MYSQL_FLAVOR" in
@@ -93,7 +74,6 @@ for uid_index in $uids; do
   $VTROOT/bin/mysqlctl \
     -log_dir $VTDATAROOT/tmp \
     -tablet_uid $uid \
-    $dbconfig_dba_flags \
     -mysql_port $mysql_port \
     $action &
 done
@@ -142,7 +122,6 @@ for uid_index in $uids; do
     -pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \
     -vtctld_addr http://$hostname:$vtctld_web_port/ \
     $optional_auth_args \
-    $dbconfig_flags \
     > $VTDATAROOT/$tablet_dir/vttablet.out 2>&1 &
 
   echo "Access tablet $alias at http://$hostname:$port/debug/status"