tinyspeck · demmer · Oct 3, 2017 · Sep 2, 2017 · Sep 3, 2017 · Sep 2, 2017
diff --git a/.pullapprove.yml b/.pullapprove.yml
@@ -14,16 +14,15 @@ groups:
       - alainjobart
       - alainjobart-bot
       - AndyDiamondstein
+      - bbeaudreault
+      - demmer
       - enisoc
       - enisoc-bot
-      - erzel
-      - erzel-bot
+      - harshit-gangal
       - mberlin-bot
       - michael-berlin
-      - sougou
-      - sougou-bot
-      - thompsonja
-      - thompsonja-bot
       - pivanof
       - pivanof-bot
+      - sougou
+      - sougou-bot
 
diff --git a/.ruby-version b/.ruby-version
@@ -0,0 +1 @@
+2.2.3
diff --git a/.travis.yml b/.travis.yml
@@ -1,152 +1,61 @@
-# TODO(mberlin): Travis will break this configuration on Sep 5, 2017.
-# Moving off the container based infrastructure (sudo: required) may buy us more time.
-# But we should move to Docker based tests eventually.
-# See their announcement: https://blog.travis-ci.com/2017-07-11-trusty-as-default-linux-is-coming
-dist: precise
-# Use container-based infrastructure (see: http://docs.travis-ci.com/user/workers/container-based-infrastructure/).
-sudo: false
+# Travis CI configuration for Vitess.
+#
+# Note that we have our own test runner written in Go (test.go) which downloads
+# our bootstrap Docker image and runs all tests within Docker.
+# This solution is slower than running the tests natively, but has the
+# advantage that we do not have to install (and cache) any dependencies within
+# Travis itself.
+#
+# For the record, we expect the following overhead per Travis build:
+# - 20 seconds Travis starting up the VM.
+# - Up to 2 minutes to pull the Docker image.
+# - More than a minute to run "make build" and cache the result as temporary
+#   Docker image.
+#
+# In total, it will always take up to 4 minutes until the environment is
+# bootstrapped and the first test can be run.
+#
+# Open TODOs:
+# - Re-add travis/check_make_proto.sh, ideally as part of test/config.json.
+# - Add a presubmit which checks that vendor/vendor.json is the same as in the Docker image. This will prevent people from making changes to it without pushing new bootstrap Docker images.
+
+# sudo is required because we run Docker in our builds.
+# See: https://docs.travis-ci.com/user/docker/
+sudo: required
+
+services:
+  - docker
+
 language: go
 go:
-  - 1.8
+  - 1.9
 go_import_path: github.com/youtube/vitess
-addons:
-  apt:
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      # NOTE: When you add a dependency, don't forget to add comment why it's necessary.
-      - automake
-      - libtool
-      - php5-cli
-      - php5-dev
-      - python-dev
-      - python-mysqldb
-      - python-pip
-      - libssl-dev
-      - g++-4.8
-      - gcc-4.8
-      - git
-      - pkg-config
-      - python-virtualenv
-      # installs libaio1 which is required by MariaDB 10.0 server package
-      - libaio-dev
-      # required by travis script below to measure CPU and memory usage
-      - time
-      # Optional dependency. Without a running syslog daemon, Vitess will keep complaining that it could not log events and spam the logs.
-      - rsyslog
-# Cache directories of dependencies which are built by bootstrap.sh
-cache:
-  directories:
-    # Cache downloaded and extracted MariaDB 10.0 packages.
-    - $MYSQL_ROOT
-    # Cache bootstrapped dependencies (e.g. protobuf and gRPC).
-    - $HOME/gopath/dist/etcd
-    - $HOME/gopath/dist/grpc/.build_finished
-    - $HOME/gopath/dist/grpc/usr/local
-    - $HOME/gopath/dist/py-mock-1.0.1/.build_finished
-    - $HOME/gopath/dist/py-mock-1.0.1/lib/python2.7/site-packages
-    - $HOME/gopath/dist/vt-zookeeper-3.4.6/.build_finished
-    - $HOME/gopath/dist/vt-zookeeper-3.4.6/lib
-    # Maven repository for Java dependencies.
-    - $HOME/.m2
-    - $HOME/.phpenv
-before_cache:
-  # Travis CI caching doesn't work with these symlinks. Just delete them.
-  - rm $HOME/gopath/bin/etcd
-  - rm $HOME/gopath/bin/zksrv.sh
-  # Delete these files because they keep changing (having the latest timestamp
-  # of an update) and trigger a lengthy update of the cache (~25 seconds).
-  - rm $HOME/.m2/repository/io/vitess/*/maven-metadata-local.xml
-  - rm $HOME/.m2/repository/io/vitess/*/*-SNAPSHOT/*-SNAPSHOT*.jar
-  - rm $HOME/.m2/repository/io/vitess/*/*-SNAPSHOT/_remote.repositories
-  - rm $HOME/.m2/repository/io/vitess/*/*-SNAPSHOT/maven-metadata-local.xml
-  - rm $HOME/.m2/repository/io/vitess/*/*-SNAPSHOT/resolver-status.properties
-  - rm $HOME/.m2/repository/io/grpc/grpc-core/resolver-status.properties
-  - rm $HOME/.m2/repository/io/grpc/protoc-gen-grpc-java/*/protoc-gen-grpc-java-*.pom.lastUpdated
-  - rm $HOME/.m2/repository/io/netty/netty-codec-http2/resolver-status.properties
-  # Don't cache unnecessary PHP files.
-  - rm $HOME/.phpenv/versions/*/sbin/*
-  - rm $HOME/.phpenv/versions/*/bin/php-cgi
-  - rm $HOME/.phpenv/shims/php-cgi
-  - find $HOME/.phpenv/vendor/grpc/grpc -mindepth 1 -maxdepth 1 ! -name src | xargs rm -rf
-  - find $HOME/.phpenv/vendor/grpc/grpc/src -mindepth 1 -maxdepth 1 ! -name php | xargs rm -rf
 env:
   global:
-    - MYSQL_FLAVOR=MariaDB
-    - MYSQL_ROOT=$HOME/mysql
-    - VT_MYSQL_ROOT=$MYSQL_ROOT/usr
-    # Enable parallel compilation e.g. for gRPC.
-    # (The Travis CI worker is allowed to use up to 2 cores, but as of 07/2015 4 parallel compilations is actually faster.)
-    - MAKEFLAGS=-j4
     # Run go build and test with -p 4 (i.e. up to 4 packages are compiled/tested in parallel).
     # As of 07/2015 this value works best in a Travis CI container.
+    # TODO(mberlin): This will probably not be passed through to Docker. Verify and fix this.
     - VT_GO_PARALLEL_VALUE=4
-    - PATH="$HOME/.phpenv/bin:$PATH"
     # Note: The per test timeout must always be < 10 minutes because test.go
     #       does not produce any log output while running and Travis kills a
     #       build after 10 minutes without log output.
     #       See: https://docs.travis-ci.com/user/customizing-the-build#Build-Timeouts
     # To diagnose stuck tests, add "-follow" to TEST_FLAGS below. Then test.go
     # will print the test's output.
-    - TEST_FLAGS="-docker=false -timeout=8m -print-log -remote-stats=http://enisoc.com:15123/travis/stats"
-    - CC=gcc-4.8
-    - CXX=g++-4.8
-    # TODO: uncomment when php crashing is fixed
-    # - INSTALL_GRPC_PHP="$HOME/.phpenv/lib"
-    - SAUCE_USERNAME=vitess
+    - TEST_FLAGS="-docker -use_docker_cache -timeout=8m -print-log -remote-stats=http://enisoc.com:15123/travis/stats"
   matrix:
     # NOTE: Travis CI schedules up to 5 tests simultaneously.
     #       All our tests should be spread out as evenly as possible across these 5 slots.
-    #       We should always utilize all 5 slots because the cost of the setup is high (up to one minute).
-    # NOTE: Use "" if you specify a space separated list of multiple targets.
+    #       We should always utilize all 5 slots because the cost of the setup is high (up to four minutes).
     - TEST_MATRIX="-shard 0"
     - TEST_MATRIX="-shard 1"
     - TEST_MATRIX="-shard 2"
     - TEST_MATRIX="-shard 3"
     - TEST_MATRIX="-shard 4"
-before_install:
-  - travis/download_mariadb.sh
-  # TODO: uncomment when php crashing is fixed
-  # - travis/php_init.sh
-install:
-  # TODO: uncomment when php crashing is fixed
-  # - eval "$(phpenv init -)"
-  - ./bootstrap.sh
-before_script:
-  - source dev.env
-  # Travis only tests. Run only on the first shard.
-  # Part of "before_script" because in "script" the job would not fail immediately if a command fails.
-  - |
-    if [[ $TEST_MATRIX = *"-shard 0"* ]]; then
-      travis/check_make_proto.sh
-    fi
-  # Webdriver tests are only on the last shard for non PRs. Start Sauce Connect only on this shard.
-  - |
-    if [[ $TEST_MATRIX = *"-shard 4"* && $TRAVIS_PULL_REQUEST = "false" ]]; then
-      tools/sauce_connect_setup.sh
-    fi
 script:
-  # Log GOMAXPROCS (should be 2 as of 07/2015).
-  - go run travis/log_gomaxprocs.go
-  - |
-    if [[ $TRAVIS_PULL_REQUEST = "false" ]]; then
-      go run test.go $TEST_FLAGS $TEST_MATRIX
-    else
-      # Exclude webdriver tests on Travis PRs since the sauce addon does not work for PRs
-      go run test.go $TEST_FLAGS $TEST_MATRIX -exclude=webdriver
-    fi
-after_script:
-  # Stop Sauce Connect at the end
-  - |
-    if [[ $TEST_MATRIX = *"-shard 4"* && $TRAVIS_PULL_REQUEST = "false" ]]; then
-      tools/sauce_connect_teardown.sh
-    fi
-after_failure:
-  # In case of errors, output log files to make it easier to debug the error.
-  # List all available files.
-  - ls -alR $HOME/gopath/vtdataroot
-  # Output *.log* and *.stderr files. (Add -keep-data to TEST_FLAGS above or tests will delete their logs.)
-  - find $HOME/gopath/vtdataroot \( -name "*.log*" -or -name "*.stderr" \) -type f -print0 | xargs -0r --verbose --max-args=1 cat
+  - go run test.go $TEST_FLAGS $TEST_MATRIX
+  # Uncomment the next line to verify the GOMAXPROCS value (should be 2 as of 09/2017).
+  # - ./docker/test/run.sh mysql57 'go run travis/log_gomaxprocs.go'
 notifications:
   slack:
     secure: S9n4rVWuEvSaF9RZUIx3Nkc2ycpM254zmalyMMbT5EmV1Xz6Zww2FL39RR5d57zsZ2M8GVW5n9uB8Bx57mr+L/wClEltzknYr7MA2/yYNMo5iK83tdQtNNw5U+dZG9/Plhlm4n883lcw9aZOyotNcLg2zBsd48Y74olk4NdmSfo=
diff --git a/ADOPTERS.md b/ADOPTERS.md
@@ -1,11 +1,13 @@
 This is an alphabetical list of known adopters of Vitess. Some have already gone into production, and others are at various stages of testing.
 
+* [YouTube](http://youtube.com)
+* [Axon](http://axon.com)
 * [BetterCloud](http://bettercloud.com)
 * [FlipKart](http://flipkart.com)
 * [HubSpot](http://product.hubspot.com/)
 * [Nozzle](http://nozzle.io)
 * [Pixel Federation](http://pixelfederation.com)
+* [Quiz of Kings](http://quizofkings.com)
 * [Slack](http://slack.com)
 * [Square](http://square.com)
 * [Stitch Labs](http://stitchlabs.com)
-* [YouTube](http://youtube.com)
diff --git a/Makefile b/Makefile
@@ -20,7 +20,7 @@ MAKEFLAGS = -s
 
 .PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test php_test reshard_tests
 
-all: build test
+all: build
 
 # Set a custom value for -p, the number of packages to be built/tested in parallel.
 # This is currently only used by our Travis CI test configuration.
@@ -80,6 +80,10 @@ unit_test: build
 unit_test_cover: build
 	go test $(VT_GO_PARALLEL) -cover ./go/... | misc/parse_cover.py
 
+# TODO(mberlin): This is currently disabled in test/config.json for automatic
+# runs on Travis (manual=false). Re-enable it there once we switched the Travis
+# tests to using Docker and we verified that this is no longer flaky e.g. in
+# doubt increase timeouts in all tests.
 unit_test_race: build
 	tools/unit_test_race.sh
 

diff --git a/bootstrap.sh b/bootstrap.sh
@@ -21,10 +21,9 @@ fi
 
 # Run parallel make, based on number of cores available.
 case $(uname) in
-Linux)	NB_CORES=$(grep -c '^processor' /proc/cpuinfo);;
-Darwin)	NB_CORES=$(sysctl hw.ncpu | awk '{ print $2 }');;
+  Linux)  NB_CORES=$(grep -c '^processor' /proc/cpuinfo);;
+  Darwin) NB_CORES=$(sysctl hw.ncpu | awk '{ print $2 }');;
 esac
-
 if [ -n "$NB_CORES" ]; then
   export MAKEFLAGS="-j$((NB_CORES+1)) -l${NB_CORES}"
 fi
@@ -34,7 +33,7 @@ function fail() {
   exit 1
 }
 
-[ -f bootstrap.sh ] || fail "bootstrap.sh must be run from its current directory"
+[ "$(dirname $0)" = '.' ] || fail "bootstrap.sh must be run from its current directory"
 
 go version 2>&1 >/dev/null || fail "Go is not installed or is not on \$PATH"
 
@@ -49,6 +48,24 @@ mkdir -p $VTROOT/vthook
 echo "Updating git submodules..."
 git submodule update --init
 
+# Install "protoc" protobuf compiler binary.
+protoc_version=3.4.0
+protoc_dist=$VTROOT/dist/protoc
+protoc_version_file=$protoc_dist/version
+if [[ -f $protoc_version_file && "$(cat $protoc_version_file)" == "$protoc_version" ]]; then
+  echo "skipping protoc install. remove $protoc_version_file to force re-install."
+else
+  rm -rf $protoc_dist
+  mkdir -p $protoc_dist
+  download_url=https://github.com/google/protobuf/releases/download/v${protoc_version}/protoc-${protoc_version}-linux-x86_64.zip
+  (cd $protoc_dist && \
+    wget $download_url && \
+    unzip protoc-${protoc_version}-linux-x86_64.zip)
+  [ $? -eq 0 ] || fail "protoc download failed"
+  echo "$protoc_version" > $protoc_version_file
+fi
+ln -snf $protoc_dist/bin/protoc $VTROOT/bin/protoc
+
 # install zookeeper
 zk_ver=3.4.6
 zk_dist=$VTROOT/dist/vt-zookeeper-$zk_ver
@@ -268,7 +285,9 @@ selenium_dist=$VTROOT/dist/selenium
 mkdir -p $selenium_dist
 $VIRTUALENV $selenium_dist
 PIP=$selenium_dist/bin/pip
-$PIP install selenium
+# PYTHONPATH is removed for `pip install` because otherwise it can pick up go/dist/grpc/usr/local/lib/python2.7/site-packages
+# instead of go/dist/selenium/lib/python3.5/site-packages and then can't find module 'pip._vendor.requests'
+PYTHONPATH= $PIP install selenium
 mkdir -p $VTROOT/dist/chromedriver
 curl -sL http://chromedriver.storage.googleapis.com/2.25/chromedriver_linux64.zip > chromedriver_linux64.zip
 unzip -o -q chromedriver_linux64.zip -d $VTROOT/dist/chromedriver

diff --git a/composer.json b/composer.json
@@ -9,7 +9,7 @@
     "php": ">=5.5.0",
     "stanley-cheung/protobuf-php": "v0.6",
     "google/auth": "v0.10",
-    "grpc/grpc": "v1.0.0"
+    "grpc/grpc": "v1.6.0"
   },
   "autoload": {
     "psr-4": {