Switch to go modules

.travis.yml

@@ -17,7 +17,7 @@
 #
 # Open TODOs:
 # - Re-add travis/check_make_proto.sh, ideally as part of test/config.json.
-# - Add a presubmit which checks that vendor/vendor.json is the same as in the Docker image. This will prevent people from making changes to it without pushing new bootstrap Docker images.
+# - Add a presubmit which checks that if bootstrap has changed, and docker image is out of date.
 
 # sudo is required because we run Docker in our builds.
 # See: https://docs.travis-ci.com/user/docker/

Makefile

@@ -81,11 +81,6 @@ cleanall:
 	# directories created by bootstrap.sh
 	# - exclude vtdataroot and vthook as they may have data we want
 	rm -rf ../../../../bin ../../../../dist ../../../../lib ../../../../pkg
-	# keep the vendor.json file but nothing else under the vendor directory as it's not actually part of the Vitess repo
-	rm -rf vendor/cloud.google.com vendor/github.com vendor/golang.org vendor/google.golang.org vendor/gopkg.in
-	# other stuff in the go hierarchy that is not under vendor/
-	rm -rf ../../../golang.org ../../../honnef.co
-	rm -rf ../../../github.com/golang ../../../github.com/kardianos ../../../github.com/kisielk
 	# Remind people to run bootstrap.sh again
 	echo "Please run bootstrap.sh again to setup your environment"
 
@@ -117,13 +112,6 @@ java_test:
 	go install ./go/cmd/vtgateclienttest ./go/cmd/vtcombo
 	mvn -f java/pom.xml clean verify
 
-# TODO(mberlin): Remove the manual copy once govendor supports a way to
-# install vendor'd programs: https://github.com/kardianos/govendor/issues/117
-install_protoc-gen-go:
-	mkdir -p $${GOPATH}/src/github.com/golang/
-	cp -a vendor/github.com/golang/protobuf $${GOPATH}/src/github.com/golang/
-	go install github.com/golang/protobuf/protoc-gen-go
-
 # Find protoc compiler.
 # NOTE: We are *not* using the "protoc" binary (as suggested by the grpc Go
 #       quickstart for example). Instead, we run "protoc" via the Python
@@ -268,7 +256,6 @@ rebalance_tests:
 
 # Release a version.
 # This will generate a tar.gz file into the releases folder with the current source
-# as well as the vendored libs.
 release: docker_base
 	@if [ -z "$VERSION" ]; then \
 	  echo "Set the env var VERSION with the release version"; exit 1;\

bootstrap.sh

@@ -19,7 +19,6 @@
 # Outline of this file.
 # 0. Initialization and helper methods.
 # 1. Installation of dependencies.
-# 2. Installation of Go tools and vendored Go dependencies.
 
 BUILD_TESTS=${BUILD_TESTS:-1}
 BUILD_PYTHON=${BUILD_PYTHON:-1}
@@ -45,15 +44,9 @@ mkdir -p "$VTROOT/bin"
 mkdir -p "$VTROOT/lib"
 mkdir -p "$VTROOT/vthook"
 
-# Install git hooks.
-echo "creating git hooks"
-mkdir -p "$VTTOP/.git/hooks"
-ln -sf "$VTTOP/misc/git/pre-commit" "$VTTOP/.git/hooks/pre-commit"
-ln -sf "$VTTOP/misc/git/commit-msg" "$VTTOP/.git/hooks/commit-msg"
-(cd "$VTTOP" && git config core.hooksPath "$VTTOP/.git/hooks")
-
+# This is required for VIRTUALENV
+# Used by Python below
 
-# Set up the proper GOPATH for go get below.
 if [ "$BUILD_TESTS" == 1 ] ; then
     source ./dev.env
 else
@@ -76,6 +69,14 @@ else
     ln -snf "$VTTOP/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh"
 fi
 
+# git hooks are only required if someone intends to contribute.
+
+echo "creating git hooks"
+mkdir -p "$VTTOP/.git/hooks"
+ln -sf "$VTTOP/misc/git/pre-commit" "$VTTOP/.git/hooks/pre-commit"
+ln -sf "$VTTOP/misc/git/commit-msg" "$VTTOP/.git/hooks/commit-msg"
+(cd "$VTTOP" && git config core.hooksPath "$VTTOP/.git/hooks")
+
 # install_dep is a helper function to generalize the download and installation of dependencies.
 #
 # If the installation is successful, it puts the installed version string into
@@ -273,7 +274,6 @@ if [ "$BUILD_PYTHON" == 1 ] ; then
     install_dep "Selenium" "latest" "$VTROOT/dist/selenium" install_selenium
 fi
 
-
 # Download chromedriver (necessary to run test/vtctld_web_test.py).
 function install_chromedriver() {
   local version="$1"
@@ -291,43 +291,5 @@ if [ "$BUILD_PYTHON" == 1 ] ; then
   PYTHONPATH='' $PIP install mysql-connector-python
 fi
 
-#
-# 2. Installation of Go tools and vendored Go dependencies.
-#
-
-
-# Install third-party Go tools used as part of the development workflow.
-#
-# DO NOT ADD LIBRARY DEPENDENCIES HERE. Instead use govendor as described below.
-#
-# Note: We explicitly do not vendor the tools below because a) we want to stay
-# on their latest version and b) it's easier to "go install" them this way.
-gotools=" \
-       github.com/golang/mock/mockgen \
-       github.com/kardianos/govendor \
-       golang.org/x/lint/golint \
-       golang.org/x/tools/cmd/cover \
-       golang.org/x/tools/cmd/goimports \
-       golang.org/x/tools/cmd/goyacc \
-"
-echo "Installing dev tools with 'go get'..."
-# shellcheck disable=SC2086
-go get -u $gotools || fail "Failed to download some Go tools with 'go get'. Please re-run bootstrap.sh in case of transient errors."
-
-# Download dependencies that are version-pinned via govendor.
-#
-# To add a new dependency, run:
-#   govendor fetch <package_path>
-#
-# Existing dependencies can be updated to the latest version with 'fetch' as well.
-#
-# Then:
-#   git add vendor/vendor.json
-#   git commit
-#
-# See https://github.com/kardianos/govendor for more options.
-echo "Updating govendor dependencies..."
-govendor sync || fail "Failed to download/update dependencies with govendor. Please re-run bootstrap.sh in case of transient errors."
-
 echo
 echo "bootstrap finished - run 'source dev.env' or 'source build.env' in your shell before building."

build.env

@@ -36,3 +36,5 @@ export VTROOT
 if [[ "$VTTOP" == "${VTTOP/\/src\/vitess.io\/vitess/}" ]]; then
   echo "WARNING: VTTOP($VTTOP) does not contain src/vitess.io/vitess"
 fi
+
+export GO111MODULE=on

dev.env

@@ -98,9 +98,6 @@ fi
 PKG_CONFIG_PATH=$(prepend_path "$PKG_CONFIG_PATH" "$VTROOT/lib")
 export PKG_CONFIG_PATH
 
-GOPATH=$(prepend_path "$GOPATH" "$VTROOT")
-export GOPATH
-
 # Useful aliases. Remove if inconvenient.
 alias gt='cd $GOTOP'
 alias pt='cd $PYTOP'

docker/bootstrap/Dockerfile.common

@@ -46,25 +46,19 @@ ENV VTDATAROOT $VTROOT/vtdataroot
 ENV VTPORTSTART 15000
 ENV PYTHONPATH $VTROOT/dist/grpc/usr/local/lib/python2.7/site-packages:$VTROOT/dist/py-mock-1.0.1/lib/python2.7/site-packages:$VTROOT/py-vtdb:$VTROOT/dist/selenium/lib/python2.7/site-packages
 ENV GOBIN $VTROOT/bin
-ENV GOPATH $VTROOT
 ENV PATH $VTROOT/bin:$VTROOT/dist/maven/bin:$VTROOT/dist/chromedriver:$PATH
-ENV VT_MYSQL_ROOT /usr
 ENV PKG_CONFIG_PATH $VTROOT/lib
 ENV USER vitess
+ENV GO111MODULE on
 
 # Copy files needed for bootstrap
 COPY bootstrap.sh dev.env build.env /vt/src/vitess.io/vitess/
 COPY config /vt/src/vitess.io/vitess/config
 COPY third_party /vt/src/vitess.io/vitess/third_party
 COPY tools /vt/src/vitess.io/vitess/tools
 COPY travis /vt/src/vitess.io/vitess/travis
-COPY vendor/vendor.json /vt/src/vitess.io/vitess/vendor/
 
-# Download vendored Go dependencies
-RUN cd /vt/src/vitess.io/vitess && \
-    go get -u github.com/kardianos/govendor && \
-    govendor sync && \
-    rm -rf /vt/.cache
+RUN go mod download
 
 # Create vitess user
 RUN groupadd -r vitess && useradd -r -g vitess vitess && \

docker/test/run.sh

@@ -176,12 +176,12 @@ copy_src_cmd="cp -R /tmp/src/!(vendor|bootstrap.sh) ."
 # Git repository.
 copy_src_cmd=$(append_cmd "$copy_src_cmd" "cp -R /tmp/src/.git .")
 
-# Copy vendor/vendor.json file if it changed
-run_bootstrap_cmd="if [[ \$(diff -w vendor/vendor.json /tmp/src/vendor/vendor.json) ]]; then cp -f /tmp/src/vendor/vendor.json vendor/; sync_vendor=1; fi"
+# Enable gomodules
+run_bootstrap_cmd="export GO111MODULE=on"
 # Copy bootstrap.sh if it changed
 run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi")
 # run bootstrap.sh if necessary
-run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; else if [[ -n \$sync_vendor ]]; then govendor sync; fi; fi")
+run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi")
 copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd")
 
 # Construct the command we will actually run.

go.mod

@@ -0,0 +1,77 @@
+module vitess.io/vitess
+
+go 1.12
+
+require (
+	cloud.google.com/go v0.43.0
+	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
+	github.com/aws/aws-sdk-go v0.0.0-20180223184012-ebef4262e06a
+	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 // indirect
+	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
+	github.com/coreos/etcd v0.0.0-20170626015032-703663d1f6ed
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
+	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/ghodss/yaml v0.0.0-20161207003320-04f313413ffd // indirect
+	github.com/go-ini/ini v1.12.0 // indirect
+	github.com/gogo/protobuf v1.2.1 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
+	github.com/golang/mock v1.3.1
+	github.com/golang/protobuf v1.3.2
+	github.com/golang/snappy v0.0.0-20170215233205-553a64147049
+	github.com/gorilla/websocket v0.0.0-20160912153041-2d1e4548da23
+	github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190118093823-f849b5445de4
+	github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20180418170936-39de4380c2e0
+	github.com/grpc-ecosystem/grpc-gateway v0.0.0-20161128002007-199c40a060d1 // indirect
+	github.com/hashicorp/consul v1.4.0
+	github.com/hashicorp/go-msgpack v0.5.5 // indirect
+	github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 // indirect
+	github.com/hashicorp/go-uuid v1.0.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.3 // indirect
+	github.com/hashicorp/memberlist v0.1.4 // indirect
+	github.com/hashicorp/serf v0.0.0-20161207011743-d3a67ab21bc8 // indirect
+	github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect
+	github.com/jonboulle/clockwork v0.1.0 // indirect
+	github.com/klauspost/compress v0.0.0-20180801095237-b50017755d44 // indirect
+	github.com/klauspost/cpuid v1.2.0 // indirect
+	github.com/klauspost/crc32 v1.2.0 // indirect
+	github.com/klauspost/pgzip v1.2.0
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/mattn/go-runewidth v0.0.1 // indirect
+	github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1
+	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
+	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/olekukonko/tablewriter v0.0.0-20160115111002-cca8bbc07984
+	github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02
+	github.com/opentracing/opentracing-go v1.1.0
+	github.com/pborman/uuid v0.0.0-20160824210600-b984ec7fa9ff
+	github.com/prometheus/client_golang v0.9.2
+	github.com/satori/go.uuid v0.0.0-20160713180306-0aa62d5ddceb // indirect
+	github.com/stretchr/testify v1.4.0
+	github.com/tchap/go-patricia v0.0.0-20160729071656-dd168db6051b
+	github.com/uber-go/atomic v1.4.0 // indirect
+	github.com/uber/jaeger-client-go v2.16.0+incompatible
+	github.com/uber/jaeger-lib v2.0.0+incompatible // indirect
+	github.com/ugorji/go v1.1.7 // indirect
+	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	github.com/z-division/go-zookeeper v0.0.0-20190128072838-6d7457066b9b
+	go.uber.org/atomic v1.4.0 // indirect
+	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472
+	golang.org/x/lint v0.0.0-20190409202823-959b441ac422
+	golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+	golang.org/x/sys v0.0.0-20190830142957-1e83adbbebd0 // indirect
+	golang.org/x/text v0.3.2
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
+	golang.org/x/tools v0.0.0-20190830154057-c17b040389b9
+	google.golang.org/api v0.7.0
+	google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64 // indirect
+	google.golang.org/grpc v1.21.1
+	gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/ldap.v2 v2.5.0
+	honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a
+)