diff --git a/bootstrap.sh b/bootstrap.sh index d5a24af6550..3c58a603efc 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -177,17 +177,16 @@ function install_zookeeper() { local dist="$2" zk="zookeeper-$version" - wget "http://apache.org/dist/zookeeper/$zk/$zk.tar.gz" + wget "https://apache.org/dist/zookeeper/$zk/$zk.tar.gz" tar -xzf "$zk.tar.gz" + ant -f "$zk/build.xml" package + ant -f "$zk/zookeeper-contrib/zookeeper-contrib-fatjar/build.xml" jar mkdir -p lib - cp "$zk/contrib/fatjar/$zk-fatjar.jar" lib - # TODO(sougou): when version changes, see if we can drop the 'zip -d' hack to get the fatjars working. - # If yes, also delete "zip" from the Dockerfile files and the manual build instructions again. - # 3.4.13 workaround: Delete META-INF files which should not be in there. - zip -d "lib/$zk-fatjar.jar" 'META-INF/*.SF' 'META-INF/*.RSA' 'META-INF/*SF' + cp "$zk/build/zookeeper-contrib/zookeeper-contrib-fatjar/zookeeper-dev-fatjar.jar" "lib/$zk-fatjar.jar" + zip -d "lib/$zk-fatjar.jar" 'META-INF/*.SF' 'META-INF/*.RSA' 'META-INF/*SF' || true # needed for >=3.4.10 <3.5 rm -rf "$zk" "$zk.tar.gz" } -zk_ver=3.4.13 +zk_ver=${ZK_VERSION:-3.4.14} install_dep "Zookeeper" "$zk_ver" "$VTROOT/dist/vt-zookeeper-$zk_ver" install_zookeeper @@ -332,7 +331,7 @@ if [ "$BUILD_TESTS" == 1 ] ; then echo "Found MySQL 5.6+ installation in $VT_MYSQL_ROOT." ;; - "MariaDB") + "MariaDB" | "MariaDB103" ) myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)" [[ "$myversion" =~ MariaDB ]] || fail "Couldn't find MariaDB in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location." echo "Found MariaDB installation in $VT_MYSQL_ROOT." @@ -364,7 +363,7 @@ if [ "$BUILD_TESTS" == 1 ] ; then echo "bootstrap finished - run 'source dev.env' in your shell before building." else echo - echo "bootstrap finished - run 'source build.env' in your shell before building." + echo "bootstrap finished - run 'source build.env' in your shell before building." fi diff --git a/doc/APIScope.md b/doc/APIScope.md index 2442ff58e84..f51c7e51a94 100644 --- a/doc/APIScope.md +++ b/doc/APIScope.md @@ -31,7 +31,7 @@ The conventions Vitess follow is: ## Execute API -The main entry point of a Vitess cluster is the 'Execute' call (or StreamExecute +The main entry point of a Vitess cluster is the `Execute` call (or StreamExecute for streaming queries). It takes a keyspace, a tablet type, and a query. The VSchema helps Vitess route the query to the right shard and tablet type. This is the most transparent way of accessing Vitess. Keyspace is the database, and diff --git a/doc/AdvancedFeaturesIndex.md b/doc/AdvancedFeaturesIndex.md index 0c202e60393..c2fe3a753d2 100644 --- a/doc/AdvancedFeaturesIndex.md +++ b/doc/AdvancedFeaturesIndex.md @@ -1,6 +1,6 @@ # Advanced Features -The pages below this navigation entry "Advanced Features" can be understood as an addition to the "User Guide". Here we describe advanced Vitess features which you may want to enable or tune in a production setup. +The pages below this navigation entry `Advanced Features` can be understood as an addition to the `User Guide`. Here we describe advanced Vitess features which you may want to enable or tune in a production setup. As of October 2017, many of these features are not documented yet. We plan to add pages for them later. diff --git a/doc/CodeReviews.md b/doc/CodeReviews.md index 5ffd9130bc3..61e13385320 100644 --- a/doc/CodeReviews.md +++ b/doc/CodeReviews.md @@ -25,7 +25,7 @@ We usually check these kinds of things while skimming through `git diff --cached * Look for files that shouldn't be checked in (temporary/generated files). * Googlers only: Remove Google confidential info (e.g. internal URLs). * Look for temporary code/comments you added while debugging. - * Example: fmt.Println("AAAAAAAAAAAAAAAAAA") + * Example: fmt.Println(`AAAAAAAAAAAAAAAAAA`) * Look for inconsistencies in indentation. * Use 2 spaces in everything except Go. * In Go, just use goimports. diff --git a/doc/Contributing.md b/doc/Contributing.md index 1e1a662046a..70bf486e408 100644 --- a/doc/Contributing.md +++ b/doc/Contributing.md @@ -30,7 +30,7 @@ To make sure you're writing idiomatic Go code, please read the following documen * Go Readability slides: https://talks.golang.org/2014/readability.slide * Talk about Go readability with many specific examples. -* "Effective Go": https://golang.org/doc/effective_go.html +* `Effective Go`: https://golang.org/doc/effective_go.html * Recommendations for writing good Go code. * Go Code Review Comments: https://github.com/golang/go/wiki/CodeReviewComments * The closest thing to a style guide. diff --git a/doc/DesignDocs.md b/doc/DesignDocs.md index f4d06fa0f59..0fa0e422cd6 100644 --- a/doc/DesignDocs.md +++ b/doc/DesignDocs.md @@ -1,4 +1,4 @@ -The pages below this navigation entry "Design Docs" represent the design considerations +The pages below this navigation entry `Design Docs` represent the design considerations that went behind some of the features implemented. They may not be necessarily up-to-date. Also, some of the ideas here may just be experimental proposals, and it's possible diff --git a/doc/DockerBuild.md b/doc/DockerBuild.md index bb88d3c7b5d..986b8e672c6 100644 --- a/doc/DockerBuild.md +++ b/doc/DockerBuild.md @@ -12,7 +12,7 @@ Another alternative is to customize our Docker images and build them yourselves. This is described below and involves building the `base` image first. Then you can run our build script for the `lite` image which extracts the Vitess binaries from the built `base` image. -1. Install [Docker](https://www.docker.com/) on your workstation. +1. Install [Docker](https://docs.docker.com/v17.12/install/) on your workstation. Our scripts also assume you can run the `docker` command without `sudo`, which you can do by [setting up a docker group](https://docs.docker.com/engine/installation/linux/ubuntulinux/#create-a-docker-group). @@ -22,8 +22,7 @@ Then you can run our build script for the `lite` image which extracts the Vitess 1. Go to your `src/vitess.io/vitess` directory. -1. Usually, you won't need to [build your own bootstrap image] - (https://github.com/vitessio/vitess/blob/master/docker/bootstrap/README.md) +1. Usually, you won't need to [build your own bootstrap image](https://github.com/vitessio/vitess/blob/master/docker/bootstrap/README.md) unless you edit [bootstrap.sh](https://github.com/vitessio/vitess/blob/master/bootstrap.sh) or [vendor.json](https://github.com/vitessio/vitess/blob/master/vendor/vendor.json), for example to add new dependencies. If you do need it then build the diff --git a/doc/GettingStarted.md b/doc/GettingStarted.md index 12737aca7af..f8e42628b5e 100644 --- a/doc/GettingStarted.md +++ b/doc/GettingStarted.md @@ -6,8 +6,7 @@ If you run into issues or have questions, please post on our ## Docker Build -To run Vitess in Docker, you can either use our pre-built images on [Docker Hub] -(https://hub.docker.com/u/vitess/), or build them yourself. +To run Vitess in Docker, you can either use our pre-built images on [Docker Hub](https://hub.docker.com/u/vitess/), or build them yourself. ### Docker Hub Images @@ -79,7 +78,7 @@ In addition, Vitess requires the software and libraries listed below. [MySQL 5.6](https://dev.mysql.com/downloads/mysql). You can use any installation method (src/bin/rpm/deb), but be sure to include the client development headers (`libmariadbclient-dev` or `libmysqlclient-dev`). - + The Vitess development team currently tests against MariaDB 10.0.21 and MySQL 5.6.27. @@ -117,7 +116,7 @@ In addition, Vitess requires the software and libraries listed below. 4. Select a lock service from the options listed below. It is technically possible to use another lock server, but plugins currently exist only for ZooKeeper, etcd and consul. - - ZooKeeper 3.4.13 is included by default. + - ZooKeeper 3.4.14 is included by default. - [Install etcd v3.0+](https://github.com/coreos/etcd/releases). If you use etcd, remember to include the `etcd` command on your path. @@ -152,28 +151,28 @@ In addition, Vitess requires the software and libraries listed below. ``` sh $ sudo apt-get install openjdk-7-jre ``` - + #### OS X 1. [Install Homebrew](https://brew.sh/). If your /usr/local directory is not empty and you never used Homebrew before, - it will be - [mandatory](https://github.com/Homebrew/homebrew/blob/master/share/doc/homebrew/El_Capitan_and_Homebrew.md) + it will be + [mandatory](https://github.com/Homebrew/homebrew/blob/master/share/doc/homebrew/El_Capitan_and_Homebrew.md) to run the following command: - + ``` sh sudo chown -R $(whoami):admin /usr/local ``` 2. On OS X, MySQL 5.6 has to be used, MariaDB doesn't work for some reason yet. It should be installed from Homebrew - (install steps are below). - -3. If Xcode is installed (with Console tools, which should be bundled automatically since the 7.1 version), all + (`install steps are below`). + +3. If Xcode is installed (with Console tools, which should be bundled automatically since the 7.1 version), all the dev dependencies should be satisfied in this step. If no Xcode is present, it is necessary to install pkg-config. - + ``` sh brew install pkg-config ``` - + 4. ZooKeeper is used as lock service. 5. Run the following commands: @@ -186,23 +185,23 @@ In addition, Vitess requires the software and libraries listed below. pip install tox ``` - + 6. The Vitess bootstrap script makes some checks for the go runtime, so it is recommended to have the following commands in your ~/.profile or ~/.bashrc or ~/.zshrc or ~/.bash_profile: - + ``` sh export PATH="/usr/local/opt/mysql@5.6/bin:$PATH" export PATH=/usr/local/go/bin:$PATH export GOROOT=/usr/local/go ``` - + 7. For the Vitess hostname resolving functions to work correctly, a new entry has to be added into the /etc/hosts file with the current LAN IP address of the computer (preferably IPv4) and the current hostname, which you get by typing the 'hostname' command in the terminal. - - It is also a good idea to put the following line to [force the Go DNS resolver](https://golang.org/doc/go1.5#net) + + It is also a good idea to put the following line to [force the Go DNS resolver](https://golang.org/doc/go1.5#net) in your ~/.profile or ~/.bashrc or ~/.zshrc: - + ``` export GODEBUG=netdns=go ``` @@ -211,7 +210,7 @@ In addition, Vitess requires the software and libraries listed below. 1. Navigate to the directory where you want to download the Vitess source code and clone the Vitess Github repo. After doing so, - navigate to the `src/vitess.io/vitess` directory. For go to work + navigate to the `src/vitess.io/vitess` directory. For go to work correctly, you should create a symbolic link to this inide your ${HOME}/go/src ``` sh @@ -238,7 +237,7 @@ In addition, Vitess requires the software and libraries listed below. ``` sh # export VT_MYSQL_ROOT=/usr/local/mysql - + # on OS X, this is the correct value: export VT_MYSQL_ROOT=/usr/local/opt/mysql@5.6 ``` @@ -255,7 +254,7 @@ In addition, Vitess requires the software and libraries listed below. If your machine requires a proxy to access the Internet, you will need to set the usual environment variables (e.g. `http_proxy`, `https_proxy`, `no_proxy`). - + Run the boostrap.sh script: ``` sh @@ -361,7 +360,7 @@ lock service. ZooKeeper is included in the Vitess distribution. Some Linux distributions ship with default file descriptor limits that are too low for database servers. This issue could show up - as the database crashing with the message "too many open files". + as the database crashing with the message `too many open files`. Check the system-wide `file-max` setting as well as user-specific `ulimit` values. We recommend setting them above 100K to be safe. @@ -415,7 +414,7 @@ lock service. ZooKeeper is included in the Vitess distribution. turn is configured to point to the local instance. In our sample scripts, they are both hosted in the same ZooKeeper service. - If you want to use Etcd as a distributed lock service, The following script + If you want to use Etcd as a distributed lock service, The following script creates a Etcd instance: ``` sh @@ -425,7 +424,7 @@ lock service. ZooKeeper is included in the Vitess distribution. ### example output: # enter etcd2 env # etcdmain: etcd Version: 3.X.X - # ... + # ... # etcd start done... ``` @@ -463,9 +462,8 @@ lock service. ZooKeeper is included in the Vitess distribution. 1. **Start vttablets** The `vttablet-up.sh` script brings up three vttablets, and assigns them to - a [keyspace]({% link overview/concepts.md %}#keyspace) and [shard] - ({% link overview/concepts.md %}#shard) according to the variables - set at the top of the script file. + a [keyspace]({% link overview/concepts.md %}#keyspace) and [shard]({% link overview/concepts.md %}#shard) + according to the variables set at the top of the script file. ``` sh vitess/examples/local$ ./vttablet-up.sh @@ -485,9 +483,8 @@ lock service. ZooKeeper is included in the Vitess distribution. see a keyspace named `test_keyspace` with a single shard named `0`. This is what an unsharded keyspace looks like. - If you click on the shard box, you'll see a list of [tablets] - ({% link overview/concepts.md %}#tablet) in that shard. - Note that it's normal for the tablets to be unhealthy at this point, since + If you click on the shard box, you'll see a list of [tablets]({% link overview/concepts.md %}#tablet) + in that shard. Note that it's normal for the tablets to be unhealthy at this point, since you haven't initialized them yet. You can also click the **STATUS** link on each tablet to be taken to its @@ -586,11 +583,11 @@ lock service. ZooKeeper is included in the Vitess distribution. In the examples, we are just using a single database with no specific configuration. So we just need to make that (empty) configuration visible for serving. This is done by running the following command: - + ``` sh vitess/examples/local$ ./lvtctl.sh RebuildVSchemaGraph ``` - + (As it works, this command will not display any output.) 1. **Start vtgate** diff --git a/doc/GettingStartedKubernetes.md b/doc/GettingStartedKubernetes.md index cdc311cd585..d38e7946283 100644 --- a/doc/GettingStartedKubernetes.md +++ b/doc/GettingStartedKubernetes.md @@ -58,8 +58,7 @@ you must have a GCE account with billing enabled. The instructions below explain how to enable billing and how to associate a billing account with a project in the Google Developers Console. -1. Log in to the Google Developers Console to [enable billing] - (https://console.developers.google.com/billing). +1. Log in to the Google Developers Console to [enable billing](https://console.developers.google.com/billing). 1. Click the **Billing** pane if you are not there already. 1. Click **New billing account**. 1. Assign a name to the billing account -- e.g. "Vitess on @@ -80,17 +79,15 @@ account with a project in the Google Developers Console. (Both should be listed under "Google Cloud APIs".) For each, click on it, then click the **"Enable API"** button. -1. Follow the [Google Cloud SDK quickstart instructions] - (https://cloud.google.com/sdk/#Quick_Start) to set up - and test the Google Cloud SDK. You will also set your default project +1. Follow the [Google Cloud SDK quickstart instructions](https://cloud.google.com/sdk/#Quick_Start) + to set up and test the Google Cloud SDK. You will also set your default project ID while completing the quickstart. **Note:** If you skip the quickstart guide because you've previously set up the Google Cloud SDK, just make sure to set a default project ID by running the following command. Replace `PROJECT` with the project ID assigned to your [Google Developers Console](https://console.developers.google.com/) - project. You can [find the ID] - (https://cloud.google.com/compute/docs/projects#projectids) + project. You can [find the ID](https://cloud.google.com/compute/docs/projects#projectids) by navigating to the **Overview** page for the project in the Console. ``` sh @@ -196,8 +193,7 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl Then provide the mount path to the configure script here. Direct support for other cloud blob stores like Amazon S3 can be added by - implementing the Vitess [BackupStorage plugin interface] - (https://github.com/vitessio/vitess/blob/master/go/vt/mysqlctl/backupstorage/interface.go). + implementing the Vitess [BackupStorage plugin interface](https://github.com/vitessio/vitess/blob/master/go/vt/mysqlctl/backupstorage/interface.go). Let us know on the [discussion forum](https://groups.google.com/forum/#!forum/vitess) if you have any specific plugin requests. @@ -277,8 +273,7 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl 1. **Access vtctld web UI** - To access vtctld from outside Kubernetes, use [kubectl proxy] - (https://kubernetes.io/docs/tasks/access-kubernetes-api/http-proxy-access-api/) + To access vtctld from outside Kubernetes, use [kubectl proxy](https://kubernetes.io/docs/tasks/access-kubernetes-api/http-proxy-access-api/) to create an authenticated tunnel on your workstation: **Note:** The proxy command runs in the foreground, @@ -294,8 +289,7 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl http://localhost:8001/api/v1/namespaces/default/services/vtctld:web/proxy - You can also use this proxy to access the [Kubernetes Dashboard] - (https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/), + You can also use this proxy to access the [Kubernetes Dashboard](https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/), where you can monitor nodes, pods, and services: http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/. @@ -307,8 +301,7 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl To enable RPC access into the Kubernetes cluster, we'll again use `kubectl` to set up an authenticated tunnel. Unlike the HTTP proxy - we used for the web UI, this time we need raw [port forwarding] - (https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/) + we used for the web UI, this time we need raw [port forwarding](https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/) for vtctld's [gRPC](https://grpc.io) port. Since the tunnel needs to target a particular vtctld pod name, @@ -387,8 +380,8 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl initialized the databases on them yet. It can take some time for the tablets to come up for the first time if a pod - was scheduled on a node that hasn't downloaded the [Vitess Docker image] - (https://hub.docker.com/u/vitess/) yet. You can also check the status of the + was scheduled on a node that hasn't downloaded the [Vitess Docker image](https://hub.docker.com/u/vitess/) + yet. You can also check the status of the tablets from the command line using `kvtctl.sh`: ``` sh @@ -529,8 +522,7 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl Vitess uses [vtgate]({% link overview/index.md %}#vtgate) to route each client query to the correct `vttablet`. In Kubernetes, a `vtgate` service distributes connections to a pool of `vtgate` pods. The pods are curated by - a [replication controller] - (https://kubernetes.io/docs/concepts/workloads/controllers/replicationcontroller/). + a [replication controller](https://kubernetes.io/docs/concepts/workloads/controllers/replicationcontroller/). ``` sh vitess/examples/kubernetes$ ./vtgate-up.sh @@ -566,8 +558,7 @@ which tells Kubernetes to create a public [load balancer](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) using the API for whatever platform your Kubernetes cluster is in. -You also need to [allow access through your platform's firewall] -(https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/). +You also need to [allow access through your platform's firewall](https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/). ``` sh # For example, to open port 80 in the GCE firewall: @@ -614,8 +605,7 @@ vitess/examples/kubernetes$ ./kvtctl.sh ExecuteFetchAsDba test-0000000100 "SELEC # +------+---------------------+---------+ ``` -The [GuestBook source code] -(https://github.com/vitessio/vitess/tree/master/examples/kubernetes/guestbook) +The [GuestBook source code](https://github.com/vitessio/vitess/tree/master/examples/kubernetes/guestbook) provides more detail about how the app server interacts with Vitess. ## Try Vitess resharding @@ -745,8 +735,8 @@ vitess/examples/kubernetes$ ./kvtctl.sh ExecuteFetchAsDba test-0000000100 "SELEC # +------------+ ``` -If you need a truly direct connection to mysqld, you can [launch a shell] -(#shell-access) inside the mysql container, and then connect with the `mysql` +If you need a truly direct connection to mysqld, you can [launch a shell](#shell-access) +inside the mysql container, and then connect with the `mysql` command-line client: ``` sh diff --git a/doc/GitHubWorkflow.md b/doc/GitHubWorkflow.md index 6c7a7f22765..c81aefb69b0 100644 --- a/doc/GitHubWorkflow.md +++ b/doc/GitHubWorkflow.md @@ -116,8 +116,7 @@ After this change, you can run `git push` without arguments: Then go to the [repository page](https://github.com/vitessio/vitess) and it should prompt you to create a Pull Request from a branch you recently pushed. -You can also [choose a branch manually] -(https://github.com/vitessio/vitess/compare). +You can also [choose a branch manually](https://github.com/vitessio/vitess/compare). ## Addressing Changes diff --git a/doc/ReplicatoinLagBasedThrottlingOfTransactions.md b/doc/ReplicatoinLagBasedThrottlingOfTransactions.md index 7da827854f7..49d88978919 100644 --- a/doc/ReplicatoinLagBasedThrottlingOfTransactions.md +++ b/doc/ReplicatoinLagBasedThrottlingOfTransactions.md @@ -4,8 +4,8 @@ feature is turned on, each VTTablet master monitors the replication lag from the replicas, and based on the observed replication lag tries to rate-limit the received transactions to keep the replication lag under a configured limit. -The decision of whether to throttle a transaction is done in the "BEGIN" -statement rather than in the "COMMIT" statement to avoid having a transaction +The decision of whether to throttle a transaction is done in the `BEGIN` +statement rather than in the `COMMIT` statement to avoid having a transaction perform a lot of work just to eventually be throttled and potentially rolled-back if the open-transaction timeout is exceeded. @@ -38,5 +38,5 @@ the replication lag under the desired limit; as such the desired replication lag limit may occasionally be slightly violated. * Transactions are considered homogenous. There is currently no support -for specifying how "expensive" a transaction is. +for specifying how `expensive` a transaction is. diff --git a/doc/ScalabilityPhilosophy.md b/doc/ScalabilityPhilosophy.md index d955bbbe01e..f9ad7b5a6c2 100644 --- a/doc/ScalabilityPhilosophy.md +++ b/doc/ScalabilityPhilosophy.md @@ -15,7 +15,7 @@ run it on baremetal configs, and many users still do. If deploying in a cloud, the assignment of servers and ports is abstracted away from the administrator. On baremetal, the operator still has these responsibilities. -We provide sample configs to help you [get started on Kubernetes](/getting-started/) +We provide sample configs to help you [get started on Kubernetes](https://vitess.io/docs/tutorials/kubernetes/) since it's the most similar to Borg (the [predecessor to Kubernetes](https://kubernetes.io/blog/2015/04/borg-predecessor-to-kubernetes/) on which Vitess now runs in YouTube). If you're more familiar with alternatives like Mesos, Swarm, Nomad, or DC/OS, diff --git a/doc/ServerConfiguration.md b/doc/ServerConfiguration.md index 570fa14ac13..f95b0d76c57 100644 --- a/doc/ServerConfiguration.md +++ b/doc/ServerConfiguration.md @@ -149,7 +149,7 @@ Vitess servers are written in Go. There are a few Vitess-specific knobs that app Go, being a young language, tends to add major improvements over each version. So, the latest Go version is almost always recommended. -Note that the latest Go version may be higher than the minimum version we require for compiling the binaries (see ["Prerequisites" section in the Getting Started guide](/getting-started/#prerequisites)). +Note that the latest Go version may be higher than the minimum version we require for compiling the binaries (see ["Prerequisites" section in the Getting Started guide](https://github.com/vitessio/website/blob/master/content/docs/tutorials/kubernetes.md#prerequisites)). ### GOMAXPROCS @@ -621,7 +621,7 @@ pretty much [set up Orchestrator](https://github.com/github/orchestrator/wiki/Orchestrator-Manual) in the normal way, with just a few additions as described below. -For the [Kubernetes example](/getting-started/), we provide a +For the [Kubernetes example](https://github.com/vitessio/website/blob/master/content/docs/tutorials/kubernetes.md), we provide a [sample script](https://github.com/vitessio/vitess/blob/master/examples/kubernetes/orchestrator-up.sh) to launch Orchestrator for you with these settings applied. diff --git a/doc/TwoPhaseCommitDesign.md b/doc/TwoPhaseCommitDesign.md index 1480c90ad65..14a8efbc8f2 100644 --- a/doc/TwoPhaseCommitDesign.md +++ b/doc/TwoPhaseCommitDesign.md @@ -19,7 +19,7 @@ This document intends to address the above concerns with some practical trade-of Although MySQL supports the XA protocol, it’s been unusable due to bugs. Version 5.7 claims to have fixed them all, but the more common versions in use are 5.6 and below, and we need to make 2PC work for those versions also. Even at 5.7, we still have to contend with the chattiness of XA, and the fact that it’s unused code. -The most critical component of the 2PC protocol is the ‘Prepare’ functionality. There is actually a way to implement Prepare on top of a transactional system. This is explained in a [Vitess Blog](https://vitess.io/blog/2016-06-07-distributed-transactions-in-vitess/), which will be used as foundation for this design. +The most critical component of the 2PC protocol is the `Prepare` functionality. There is actually a way to implement Prepare on top of a transactional system. This is explained in a [Vitess Blog](https://vitess.io/blog/2016-06-07-distributed-transactions-in-vitess/), which will be used as foundation for this design. Familiarity with the blog and the [2PC algorithm](http://c2.com/cgi/wiki?TwoPhaseCommit) are required to understand the rest of the document. diff --git a/doc/VtExplain.md b/doc/VtExplain.md index 22e916b50b4..93d06a8f0c3 100644 --- a/doc/VtExplain.md +++ b/doc/VtExplain.md @@ -1,6 +1,6 @@ # VTExplain Tool -The vtexplain tool provides information about how Vitess will execute a statement (the Vitess version of MySQL "EXPLAIN"). +The vtexplain tool provides information about how Vitess will execute a statement (the Vitess version of MySQL `EXPLAIN`). ## Prerequisites diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index 6d96504e03c..c1236160481 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -27,6 +27,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins xvfb \ zip \ libz-dev \ + ant \ && rm -rf /var/lib/apt/lists/* # Install Maven 3.1+ diff --git a/docker/bootstrap/Dockerfile.mariadb103 b/docker/bootstrap/Dockerfile.mariadb103 index e626cf4f2e6..d2f7759fb48 100644 --- a/docker/bootstrap/Dockerfile.mariadb103 +++ b/docker/bootstrap/Dockerfile.mariadb103 @@ -1,11 +1,11 @@ FROM vitess/bootstrap:common -# Install MariaDB 10. -RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 &&\ - add-apt-repository 'deb [arch=amd64] http://ftp.osuosl.org/pub/mariadb/repo/10.3/debian stretch main' - apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - mariadb-server \ - libmariadbclient-dev \ +# Install MariaDB 10.3 +RUN apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 \ + && add-apt-repository 'deb [arch=amd64] http://ftp.osuosl.org/pub/mariadb/repo/10.3/debian stretch main' \ + && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + mariadb-server \ + libmariadbclient-dev \ && rm -rf /var/lib/apt/lists/* # Bootstrap Vitess diff --git a/docker/bootstrap/Dockerfile.mysql57 b/docker/bootstrap/Dockerfile.mysql57 index c4c7f5c6660..75d7d03aa17 100644 --- a/docker/bootstrap/Dockerfile.mysql57 +++ b/docker/bootstrap/Dockerfile.mysql57 @@ -1,7 +1,8 @@ FROM vitess/bootstrap:common # Install MySQL 5.7 -RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 5072E1F5 && break; done && \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && \ + for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 5072E1F5 && break; done && \ add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7' && \ apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev && \ diff --git a/docker/bootstrap/Dockerfile.percona57 b/docker/bootstrap/Dockerfile.percona57 index 371890d116f..98fa12518fe 100644 --- a/docker/bootstrap/Dockerfile.percona57 +++ b/docker/bootstrap/Dockerfile.percona57 @@ -10,7 +10,8 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r } | debconf-set-selections && \ apt-get update && \ apt-get install -y --no-install-recommends \ - percona-server-server-5.7 libperconaserverclient18.1-dev && \ + percona-server-server-5.7 \ + libperconaserverclient18.1-dev && \ rm -rf /var/lib/apt/lists/* # Bootstrap Vitess diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80 index 246c6519843..9968e1dde8f 100644 --- a/docker/bootstrap/Dockerfile.percona80 +++ b/docker/bootstrap/Dockerfile.percona80 @@ -1,7 +1,7 @@ FROM vitess/bootstrap:common # Install Percona 8.0 -RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && +RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done \ && echo 'deb http://repo.percona.com/ps-80/apt stretch main' > /etc/apt/sources.list.d/percona.list && \ { \ echo debconf debconf/frontend select Noninteractive; \ @@ -14,7 +14,8 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r libperconaserverclient21 \ percona-server-tokudb \ percona-server-rocksdb \ - bzip2 + bzip2 \ + && rm -rf /var/lib/apt/lists/* # Bootstrap Vitess WORKDIR /vt/src/vitess.io/vitess diff --git a/docker/k8s/vtexplain/Dockerfile b/docker/k8s/vtexplain/Dockerfile new file mode 100644 index 00000000000..e9aa276d648 --- /dev/null +++ b/docker/k8s/vtexplain/Dockerfile @@ -0,0 +1,18 @@ +FROM vitess/base AS base + +FROM debian:stretch-slim + +# Set up Vitess environment (just enough to run pre-built Go binaries) +ENV VTROOT /vt + +# Prepare directory structure. +RUN mkdir -p /vt/bin && mkdir -p /vtdataroot + +# Copy binaries +COPY --from=base /vt/bin/vtexplain /vt/bin/ + +# add vitess user/group and add permissions +RUN groupadd -r --gid 2000 vitess && \ + useradd -r -g vitess --uid 1000 vitess && \ + chown -R vitess:vitess /vt && \ + chown -R vitess:vitess /vtdataroot diff --git a/docker/lite/Dockerfile b/docker/lite/Dockerfile index 35538633e64..ebfdb21d5c4 100644 --- a/docker/lite/Dockerfile +++ b/docker/lite/Dockerfile @@ -1,17 +1,37 @@ -# This image is only meant to be built from within the build.sh script. -FROM debian:jessie +FROM vitess/base AS builder +FROM debian:stretch-slim AS staging + +RUN mkdir -p /vt/vtdataroot/ \ + && mkdir -p /vt/bin \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && groupadd -r vitess && useradd -r -g vitess vitess + +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/bin/mysqlctld /vt/bin/ +COPY --from=builder /vt/bin/vtctld /vt/bin/ +COPY --from=builder /vt/bin/vtctlclient /vt/bin/ +COPY --from=builder /vt/bin/vtgate /vt/bin/ +COPY --from=builder /vt/bin/vttablet /vt/bin/ +COPY --from=builder /vt/bin/vtworker /vt/bin/ + +RUN chown -R vitess:vitess /vt + +FROM debian:stretch-slim # Install dependencies -RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 5072E1F5 \ - && echo 'deb http://repo.mysql.com/apt/debian/ jessie mysql-5.7' > /etc/apt/sources.list.d/mysql.list \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 \ + && echo 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7' > /etc/apt/sources.list.d/mysql.list \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive \ apt-get install -y --no-install-recommends \ bzip2 \ - libmysqlclient18 \ + libmysqlclient20 \ mysql-client \ mysql-server \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTTOP /vt/src/vitess.io/vitess @@ -23,13 +43,11 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) -COPY lite/vt /vt - -# Create vitess user -RUN groupadd -r vitess && useradd -r -g vitess vitess && \ - mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt +COPY --from=staging /vt/ /vt/ # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot +USER vitess \ No newline at end of file diff --git a/examples/local/203_vertical_split.sh b/examples/local/203_vertical_split.sh index d5a67d6c8f7..7d00e872c88 100755 --- a/examples/local/203_vertical_split.sh +++ b/examples/local/203_vertical_split.sh @@ -33,6 +33,6 @@ source "$script_root/env.sh" -log_dir "$VTDATAROOT"/tmp \ -alsologtostderr \ -use_v3_resharding_mode \ - VerticalSplitClone -min_healthy_rdonly_tablets=1 -tables=customer,corder customer/0 + VerticalSplitClone -min_healthy_tablets=1 -tables=customer,corder customer/0 disown -a diff --git a/go/cmd/vttestserver/main.go b/go/cmd/vttestserver/main.go index a018135913a..65614d80ace 100644 --- a/go/cmd/vttestserver/main.go +++ b/go/cmd/vttestserver/main.go @@ -150,6 +150,7 @@ func parseFlags() (config vttest.Config, env vttest.Environment, err error) { "A MySQL DB snapshot file") flag.StringVar(&config.TransactionMode, "transaction_mode", "MULTI", "Transaction mode MULTI (default), SINGLE or TWOPC ") + flag.Float64Var(&config.TransactionTimeout, "queryserver-config-transaction-timeout", 0, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value") flag.Parse() diff --git a/go/cmd/zk/zkcmd.go b/go/cmd/zk/zkcmd.go index 0a93356a38e..c04579d52bb 100644 --- a/go/cmd/zk/zkcmd.go +++ b/go/cmd/zk/zkcmd.go @@ -51,6 +51,8 @@ there are some slight differences in flag handling. zk -h - provide help on overriding cell selection +zk addAuth digest user:pass + zk cat /zk/path zk cat -l /zk/path1 /zk/path2 (list filename before file data) @@ -111,18 +113,19 @@ var zconn *zk2topo.ZkConn func init() { cmdMap = map[string]cmdFunc{ - "cat": cmdCat, - "chmod": cmdChmod, - "cp": cmdCp, - "edit": cmdEdit, - "ls": cmdLs, - "rm": cmdRm, - "stat": cmdStat, - "touch": cmdTouch, - "unzip": cmdUnzip, - "wait": cmdWait, - "watch": cmdWatch, - "zip": cmdZip, + "addAuth": cmdAddAuth, + "cat": cmdCat, + "chmod": cmdChmod, + "cp": cmdCp, + "edit": cmdEdit, + "ls": cmdLs, + "rm": cmdRm, + "stat": cmdStat, + "touch": cmdTouch, + "unzip": cmdUnzip, + "wait": cmdWait, + "watch": cmdWatch, + "zip": cmdZip, } } @@ -500,6 +503,15 @@ func cmdRm(ctx context.Context, subFlags *flag.FlagSet, args []string) error { return nil } +func cmdAddAuth(ctx context.Context, subFlags *flag.FlagSet, args []string) error { + subFlags.Parse(args) + if subFlags.NArg() < 2 { + return fmt.Errorf("addAuth: expected args ") + } + scheme, auth := subFlags.Arg(0), subFlags.Arg(1) + return zconn.AddAuth(ctx, scheme, []byte(auth)) +} + func cmdCat(ctx context.Context, subFlags *flag.FlagSet, args []string) error { var ( longListing = subFlags.Bool("l", false, "long listing") diff --git a/go/vt/discovery/tablet_stats_cache.go b/go/vt/discovery/tablet_stats_cache.go index db8b6780bce..326e15e2d30 100644 --- a/go/vt/discovery/tablet_stats_cache.go +++ b/go/vt/discovery/tablet_stats_cache.go @@ -56,8 +56,8 @@ type TabletStatsCache struct { entries map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry // tsm is a helper to broadcast aggregate stats. tsm srvtopo.TargetStatsMultiplexer - // cellRegions is a cache of cell regions - cellRegions map[string]string + // cellAliases is a cache of cell aliases + cellAliases map[string]string } // tabletStatsCacheEntry is the per keyspace/shard/tabletType @@ -70,7 +70,7 @@ type tabletStatsCacheEntry struct { all map[string]*TabletStats // healthy only has the healthy ones. healthy []*TabletStats - // aggregates has the per-region aggregates. + // aggregates has the per-alias aggregates. aggregates map[string]*querypb.AggregateStats } @@ -136,7 +136,7 @@ func newTabletStatsCache(hc HealthCheck, ts *topo.Server, cell string, setListen aggregatesChan: make(chan []*srvtopo.TargetStatsEntry, 100), entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), tsm: srvtopo.NewTargetStatsMultiplexer(), - cellRegions: make(map[string]string), + cellAliases: make(map[string]string), } if setListener { @@ -196,26 +196,26 @@ func (tc *TabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStat return e } -func (tc *TabletStatsCache) getRegionByCell(cell string) string { +func (tc *TabletStatsCache) getAliasByCell(cell string) string { tc.mu.Lock() defer tc.mu.Unlock() - if region, ok := tc.cellRegions[cell]; ok { - return region + if alias, ok := tc.cellAliases[cell]; ok { + return alias } - region := topo.GetRegionByCell(context.Background(), tc.ts, cell) - tc.cellRegions[cell] = region + alias := topo.GetAliasByCell(context.Background(), tc.ts, cell) + tc.cellAliases[cell] = alias - return region + return alias } // StatsUpdate is part of the HealthCheckStatsListener interface. func (tc *TabletStatsCache) StatsUpdate(ts *TabletStats) { if ts.Target.TabletType != topodatapb.TabletType_MASTER && ts.Tablet.Alias.Cell != tc.cell && - tc.getRegionByCell(ts.Tablet.Alias.Cell) != tc.getRegionByCell(tc.cell) { - // this is for a non-master tablet in a different cell and a different region, drop it + tc.getAliasByCell(ts.Tablet.Alias.Cell) != tc.getAliasByCell(tc.cell) { + // this is for a non-master tablet in a different cell and a different alias, drop it return } @@ -280,18 +280,18 @@ func (tc *TabletStatsCache) StatsUpdate(ts *TabletStats) { tc.updateAggregateMap(ts.Target.Keyspace, ts.Target.Shard, ts.Target.TabletType, e, allArray) } -// makeAggregateMap takes a list of TabletStats and builds a per-region +// makeAggregateMap takes a list of TabletStats and builds a per-alias // AggregateStats map. func (tc *TabletStatsCache) makeAggregateMap(stats []*TabletStats) map[string]*querypb.AggregateStats { result := make(map[string]*querypb.AggregateStats) for _, ts := range stats { - region := tc.getRegionByCell(ts.Tablet.Alias.Cell) - agg, ok := result[region] + alias := tc.getAliasByCell(ts.Tablet.Alias.Cell) + agg, ok := result[alias] if !ok { agg = &querypb.AggregateStats{ SecondsBehindMasterMin: math.MaxUint32, } - result[region] = agg + result[alias] = agg } if ts.Serving && ts.LastError == nil { @@ -378,8 +378,8 @@ func (tc *TabletStatsCache) GetAggregateStats(target *querypb.Target) (*querypb. return agg, nil } } - targetRegion := tc.getRegionByCell(target.Cell) - agg, ok := e.aggregates[targetRegion] + targetAlias := tc.getAliasByCell(target.Cell) + agg, ok := e.aggregates[targetAlias] if !ok { return nil, topo.NewError(topo.NoNode, topotools.TargetIdent(target)) } diff --git a/go/vt/discovery/tablet_stats_cache_test.go b/go/vt/discovery/tablet_stats_cache_test.go index 8bc5e8caafd..635d3daca2d 100644 --- a/go/vt/discovery/tablet_stats_cache_test.go +++ b/go/vt/discovery/tablet_stats_cache_test.go @@ -17,9 +17,11 @@ limitations under the License. package discovery import ( + "context" "testing" "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/memorytopo" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -27,20 +29,32 @@ import ( // TestTabletStatsCache tests the functionality of the TabletStatsCache class. func TestTabletStatsCache(t *testing.T) { - defer topo.UpdateCellsToRegionsForTests(map[string]string{}) - topo.UpdateCellsToRegionsForTests(map[string]string{ - "cell": "region1", - "cell1": "region1", - "cell2": "region2", - }) + ts := memorytopo.NewServer("cell", "cell1", "cell2") + + cellsAlias := &topodatapb.CellsAlias{ + Cells: []string{"cell", "cell1"}, + } + + ts.CreateCellsAlias(context.Background(), "region1", cellsAlias) + + defer ts.DeleteCellsAlias(context.Background(), "region1") + + cellsAlias = &topodatapb.CellsAlias{ + Cells: []string{"cell2"}, + } + + ts.CreateCellsAlias(context.Background(), "region2", cellsAlias) + + defer ts.DeleteCellsAlias(context.Background(), "region2") // We want to unit test TabletStatsCache without a full-blown // HealthCheck object, so we can't call NewTabletStatsCache. // So we just construct this object here. tsc := &TabletStatsCache{ cell: "cell", + ts: ts, entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), - cellRegions: make(map[string]string), + cellAliases: make(map[string]string), } // empty diff --git a/go/vt/mysqlctl/permissions.go b/go/vt/mysqlctl/permissions.go index 181908a0dec..18af508f412 100644 --- a/go/vt/mysqlctl/permissions.go +++ b/go/vt/mysqlctl/permissions.go @@ -22,13 +22,15 @@ import ( tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" ) -// GetPermissions lists the permissions on the mysqld +// GetPermissions lists the permissions on the mysqld. +// The rows are sorted in primary key order to help with comparing +// permissions between tablets. func GetPermissions(mysqld MysqlDaemon) (*tabletmanagerdatapb.Permissions, error) { ctx := context.TODO() permissions := &tabletmanagerdatapb.Permissions{} // get Users - qr, err := mysqld.FetchSuperQuery(ctx, "SELECT * FROM mysql.user") + qr, err := mysqld.FetchSuperQuery(ctx, "SELECT * FROM mysql.user ORDER BY host, user") if err != nil { return nil, err } @@ -37,7 +39,7 @@ func GetPermissions(mysqld MysqlDaemon) (*tabletmanagerdatapb.Permissions, error } // get Dbs - qr, err = mysqld.FetchSuperQuery(ctx, "SELECT * FROM mysql.db") + qr, err = mysqld.FetchSuperQuery(ctx, "SELECT * FROM mysql.db ORDER BY host, db, user") if err != nil { return nil, err } diff --git a/go/vt/proto/topodata/topodata.pb.go b/go/vt/proto/topodata/topodata.pb.go index 2c9a5415926..9e22bf51309 100644 --- a/go/vt/proto/topodata/topodata.pb.go +++ b/go/vt/proto/topodata/topodata.pb.go @@ -48,7 +48,7 @@ func (x KeyspaceIdType) String() string { return proto.EnumName(KeyspaceIdType_name, int32(x)) } func (KeyspaceIdType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{0} + return fileDescriptor_topodata_5ff7209b363fe950, []int{0} } // TabletType represents the type of a given tablet. @@ -117,7 +117,7 @@ func (x TabletType) String() string { return proto.EnumName(TabletType_name, int32(x)) } func (TabletType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{1} + return fileDescriptor_topodata_5ff7209b363fe950, []int{1} } // KeyRange describes a range of sharding keys, when range-based @@ -134,7 +134,7 @@ func (m *KeyRange) Reset() { *m = KeyRange{} } func (m *KeyRange) String() string { return proto.CompactTextString(m) } func (*KeyRange) ProtoMessage() {} func (*KeyRange) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{0} + return fileDescriptor_topodata_5ff7209b363fe950, []int{0} } func (m *KeyRange) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_KeyRange.Unmarshal(m, b) @@ -184,7 +184,7 @@ func (m *TabletAlias) Reset() { *m = TabletAlias{} } func (m *TabletAlias) String() string { return proto.CompactTextString(m) } func (*TabletAlias) ProtoMessage() {} func (*TabletAlias) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{1} + return fileDescriptor_topodata_5ff7209b363fe950, []int{1} } func (m *TabletAlias) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TabletAlias.Unmarshal(m, b) @@ -260,7 +260,7 @@ func (m *Tablet) Reset() { *m = Tablet{} } func (m *Tablet) String() string { return proto.CompactTextString(m) } func (*Tablet) ProtoMessage() {} func (*Tablet) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{2} + return fileDescriptor_topodata_5ff7209b363fe950, []int{2} } func (m *Tablet) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Tablet.Unmarshal(m, b) @@ -395,7 +395,7 @@ func (m *Shard) Reset() { *m = Shard{} } func (m *Shard) String() string { return proto.CompactTextString(m) } func (*Shard) ProtoMessage() {} func (*Shard) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{3} + return fileDescriptor_topodata_5ff7209b363fe950, []int{3} } func (m *Shard) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Shard.Unmarshal(m, b) @@ -470,7 +470,7 @@ func (m *Shard_ServedType) Reset() { *m = Shard_ServedType{} } func (m *Shard_ServedType) String() string { return proto.CompactTextString(m) } func (*Shard_ServedType) ProtoMessage() {} func (*Shard_ServedType) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{3, 0} + return fileDescriptor_topodata_5ff7209b363fe950, []int{3, 0} } func (m *Shard_ServedType) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Shard_ServedType.Unmarshal(m, b) @@ -527,7 +527,7 @@ func (m *Shard_SourceShard) Reset() { *m = Shard_SourceShard{} } func (m *Shard_SourceShard) String() string { return proto.CompactTextString(m) } func (*Shard_SourceShard) ProtoMessage() {} func (*Shard_SourceShard) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{3, 1} + return fileDescriptor_topodata_5ff7209b363fe950, []int{3, 1} } func (m *Shard_SourceShard) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Shard_SourceShard.Unmarshal(m, b) @@ -600,7 +600,7 @@ func (m *Shard_TabletControl) Reset() { *m = Shard_TabletControl{} } func (m *Shard_TabletControl) String() string { return proto.CompactTextString(m) } func (*Shard_TabletControl) ProtoMessage() {} func (*Shard_TabletControl) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{3, 2} + return fileDescriptor_topodata_5ff7209b363fe950, []int{3, 2} } func (m *Shard_TabletControl) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Shard_TabletControl.Unmarshal(m, b) @@ -668,7 +668,7 @@ func (m *Keyspace) Reset() { *m = Keyspace{} } func (m *Keyspace) String() string { return proto.CompactTextString(m) } func (*Keyspace) ProtoMessage() {} func (*Keyspace) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{4} + return fileDescriptor_topodata_5ff7209b363fe950, []int{4} } func (m *Keyspace) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Keyspace.Unmarshal(m, b) @@ -727,7 +727,7 @@ func (m *Keyspace_ServedFrom) Reset() { *m = Keyspace_ServedFrom{} } func (m *Keyspace_ServedFrom) String() string { return proto.CompactTextString(m) } func (*Keyspace_ServedFrom) ProtoMessage() {} func (*Keyspace_ServedFrom) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{4, 0} + return fileDescriptor_topodata_5ff7209b363fe950, []int{4, 0} } func (m *Keyspace_ServedFrom) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Keyspace_ServedFrom.Unmarshal(m, b) @@ -783,7 +783,7 @@ func (m *ShardReplication) Reset() { *m = ShardReplication{} } func (m *ShardReplication) String() string { return proto.CompactTextString(m) } func (*ShardReplication) ProtoMessage() {} func (*ShardReplication) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{5} + return fileDescriptor_topodata_5ff7209b363fe950, []int{5} } func (m *ShardReplication) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ShardReplication.Unmarshal(m, b) @@ -822,7 +822,7 @@ func (m *ShardReplication_Node) Reset() { *m = ShardReplication_Node{} } func (m *ShardReplication_Node) String() string { return proto.CompactTextString(m) } func (*ShardReplication_Node) ProtoMessage() {} func (*ShardReplication_Node) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{5, 0} + return fileDescriptor_topodata_5ff7209b363fe950, []int{5, 0} } func (m *ShardReplication_Node) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ShardReplication_Node.Unmarshal(m, b) @@ -863,7 +863,7 @@ func (m *ShardReference) Reset() { *m = ShardReference{} } func (m *ShardReference) String() string { return proto.CompactTextString(m) } func (*ShardReference) ProtoMessage() {} func (*ShardReference) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{6} + return fileDescriptor_topodata_5ff7209b363fe950, []int{6} } func (m *ShardReference) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ShardReference.Unmarshal(m, b) @@ -913,7 +913,7 @@ func (m *ShardTabletControl) Reset() { *m = ShardTabletControl{} } func (m *ShardTabletControl) String() string { return proto.CompactTextString(m) } func (*ShardTabletControl) ProtoMessage() {} func (*ShardTabletControl) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{7} + return fileDescriptor_topodata_5ff7209b363fe950, []int{7} } func (m *ShardTabletControl) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ShardTabletControl.Unmarshal(m, b) @@ -971,7 +971,7 @@ func (m *SrvKeyspace) Reset() { *m = SrvKeyspace{} } func (m *SrvKeyspace) String() string { return proto.CompactTextString(m) } func (*SrvKeyspace) ProtoMessage() {} func (*SrvKeyspace) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{8} + return fileDescriptor_topodata_5ff7209b363fe950, []int{8} } func (m *SrvKeyspace) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SrvKeyspace.Unmarshal(m, b) @@ -1035,7 +1035,7 @@ func (m *SrvKeyspace_KeyspacePartition) Reset() { *m = SrvKeyspace_Keysp func (m *SrvKeyspace_KeyspacePartition) String() string { return proto.CompactTextString(m) } func (*SrvKeyspace_KeyspacePartition) ProtoMessage() {} func (*SrvKeyspace_KeyspacePartition) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{8, 0} + return fileDescriptor_topodata_5ff7209b363fe950, []int{8, 0} } func (m *SrvKeyspace_KeyspacePartition) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SrvKeyspace_KeyspacePartition.Unmarshal(m, b) @@ -1092,7 +1092,7 @@ func (m *SrvKeyspace_ServedFrom) Reset() { *m = SrvKeyspace_ServedFrom{} func (m *SrvKeyspace_ServedFrom) String() string { return proto.CompactTextString(m) } func (*SrvKeyspace_ServedFrom) ProtoMessage() {} func (*SrvKeyspace_ServedFrom) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{8, 1} + return fileDescriptor_topodata_5ff7209b363fe950, []int{8, 1} } func (m *SrvKeyspace_ServedFrom) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SrvKeyspace_ServedFrom.Unmarshal(m, b) @@ -1137,10 +1137,7 @@ type CellInfo struct { ServerAddress string `protobuf:"bytes,1,opt,name=server_address,json=serverAddress,proto3" json:"server_address,omitempty"` // Root is the path to store data in. It is only used when talking // to server_address. - Root string `protobuf:"bytes,2,opt,name=root,proto3" json:"root,omitempty"` - // Region is a group this cell belongs to. Used by vtgate to route traffic to - // other cells (in same region) when there is no available tablet in the current cell. - Region string `protobuf:"bytes,3,opt,name=region,proto3" json:"region,omitempty"` + Root string `protobuf:"bytes,2,opt,name=root,proto3" json:"root,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1150,7 +1147,7 @@ func (m *CellInfo) Reset() { *m = CellInfo{} } func (m *CellInfo) String() string { return proto.CompactTextString(m) } func (*CellInfo) ProtoMessage() {} func (*CellInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_topodata_e30d0a39cbb15c1e, []int{9} + return fileDescriptor_topodata_5ff7209b363fe950, []int{9} } func (m *CellInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CellInfo.Unmarshal(m, b) @@ -1184,11 +1181,44 @@ func (m *CellInfo) GetRoot() string { return "" } -func (m *CellInfo) GetRegion() string { +// CellsAlias +type CellsAlias struct { + // Cells that map to this alias + Cells []string `protobuf:"bytes,2,rep,name=cells,proto3" json:"cells,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CellsAlias) Reset() { *m = CellsAlias{} } +func (m *CellsAlias) String() string { return proto.CompactTextString(m) } +func (*CellsAlias) ProtoMessage() {} +func (*CellsAlias) Descriptor() ([]byte, []int) { + return fileDescriptor_topodata_5ff7209b363fe950, []int{10} +} +func (m *CellsAlias) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CellsAlias.Unmarshal(m, b) +} +func (m *CellsAlias) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CellsAlias.Marshal(b, m, deterministic) +} +func (dst *CellsAlias) XXX_Merge(src proto.Message) { + xxx_messageInfo_CellsAlias.Merge(dst, src) +} +func (m *CellsAlias) XXX_Size() int { + return xxx_messageInfo_CellsAlias.Size(m) +} +func (m *CellsAlias) XXX_DiscardUnknown() { + xxx_messageInfo_CellsAlias.DiscardUnknown(m) +} + +var xxx_messageInfo_CellsAlias proto.InternalMessageInfo + +func (m *CellsAlias) GetCells() []string { if m != nil { - return m.Region + return m.Cells } - return "" + return nil } func init() { @@ -1211,89 +1241,90 @@ func init() { proto.RegisterType((*SrvKeyspace_KeyspacePartition)(nil), "topodata.SrvKeyspace.KeyspacePartition") proto.RegisterType((*SrvKeyspace_ServedFrom)(nil), "topodata.SrvKeyspace.ServedFrom") proto.RegisterType((*CellInfo)(nil), "topodata.CellInfo") + proto.RegisterType((*CellsAlias)(nil), "topodata.CellsAlias") proto.RegisterEnum("topodata.KeyspaceIdType", KeyspaceIdType_name, KeyspaceIdType_value) proto.RegisterEnum("topodata.TabletType", TabletType_name, TabletType_value) } -func init() { proto.RegisterFile("topodata.proto", fileDescriptor_topodata_e30d0a39cbb15c1e) } +func init() { proto.RegisterFile("topodata.proto", fileDescriptor_topodata_5ff7209b363fe950) } -var fileDescriptor_topodata_e30d0a39cbb15c1e = []byte{ - // 1217 bytes of a gzipped FileDescriptorProto +var fileDescriptor_topodata_5ff7209b363fe950 = []byte{ + // 1218 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xe1, 0x6e, 0x1b, 0x45, 0x10, 0xe6, 0xec, 0xb3, 0x63, 0x8f, 0x1d, 0xe7, 0xba, 0xa4, 0xd5, 0xe9, 0xa0, 0x22, 0xb2, 0x54, - 0x11, 0x15, 0xe1, 0xa0, 0xb4, 0x85, 0xa8, 0x12, 0x52, 0x5d, 0xc7, 0xa5, 0x69, 0x1b, 0xc7, 0x5a, - 0x3b, 0x82, 0x22, 0xa1, 0xd3, 0xc5, 0xb7, 0x71, 0x4f, 0x39, 0xdf, 0xba, 0xbb, 0x9b, 0x48, 0xe6, - 0x15, 0xf8, 0x01, 0xfc, 0xe5, 0x0d, 0x78, 0x04, 0x9e, 0x80, 0xe7, 0x80, 0x27, 0x41, 0x3b, 0x7b, - 0x67, 0x9f, 0x1d, 0x1a, 0x52, 0x94, 0x7f, 0x33, 0xbb, 0x33, 0x73, 0x33, 0xdf, 0xcc, 0x37, 0x6b, - 0x43, 0x43, 0xf1, 0x29, 0x0f, 0x03, 0x15, 0xb4, 0xa6, 0x82, 0x2b, 0x4e, 0x2a, 0x99, 0xde, 0xdc, - 0x85, 0xca, 0x4b, 0x36, 0xa3, 0x41, 0x32, 0x66, 0x64, 0x13, 0x4a, 0x52, 0x05, 0x42, 0xb9, 0xd6, - 0x96, 0xb5, 0x5d, 0xa7, 0x46, 0x21, 0x0e, 0x14, 0x59, 0x12, 0xba, 0x05, 0x3c, 0xd3, 0x62, 0xf3, - 0x01, 0xd4, 0x86, 0xc1, 0x49, 0xcc, 0x54, 0x3b, 0x8e, 0x02, 0x49, 0x08, 0xd8, 0x23, 0x16, 0xc7, - 0xe8, 0x55, 0xa5, 0x28, 0x6b, 0xa7, 0xf3, 0xc8, 0x38, 0xad, 0x53, 0x2d, 0x36, 0xff, 0xb0, 0xa1, - 0x6c, 0xbc, 0xc8, 0x67, 0x50, 0x0a, 0xb4, 0x27, 0x7a, 0xd4, 0x76, 0x6f, 0xb7, 0xe6, 0xd9, 0xe5, - 0xc2, 0x52, 0x63, 0x43, 0x3c, 0xa8, 0xbc, 0xe1, 0x52, 0x25, 0xc1, 0x84, 0x61, 0xb8, 0x2a, 0x9d, - 0xeb, 0x64, 0x0f, 0x2a, 0x53, 0x2e, 0x94, 0x3f, 0x09, 0xa6, 0xae, 0xbd, 0x55, 0xdc, 0xae, 0xed, - 0xde, 0x5d, 0x8d, 0xd5, 0xea, 0x73, 0xa1, 0x0e, 0x83, 0x69, 0x37, 0x51, 0x62, 0x46, 0xd7, 0xa6, - 0x46, 0xd3, 0x51, 0xcf, 0xd8, 0x4c, 0x4e, 0x83, 0x11, 0x73, 0x4b, 0x26, 0x6a, 0xa6, 0x23, 0x0c, - 0x6f, 0x02, 0x11, 0xba, 0x65, 0xbc, 0x30, 0x0a, 0xd9, 0x81, 0xea, 0x19, 0x9b, 0xf9, 0x42, 0x23, - 0xe5, 0xae, 0x61, 0xe2, 0x64, 0xf1, 0xb1, 0x0c, 0x43, 0x0c, 0x63, 0xd0, 0xdc, 0x06, 0x5b, 0xcd, - 0xa6, 0xcc, 0xad, 0x6c, 0x59, 0xdb, 0x8d, 0xdd, 0xcd, 0xd5, 0xc4, 0x86, 0xb3, 0x29, 0xa3, 0x68, - 0x41, 0xb6, 0xc1, 0x09, 0x4f, 0x7c, 0x5d, 0x91, 0xcf, 0x2f, 0x98, 0x10, 0x51, 0xc8, 0xdc, 0x2a, - 0x7e, 0xbb, 0x11, 0x9e, 0xf4, 0x82, 0x09, 0x3b, 0x4a, 0x4f, 0x49, 0x0b, 0x6c, 0x15, 0x8c, 0xa5, - 0x0b, 0x58, 0xac, 0x77, 0xa9, 0xd8, 0x61, 0x30, 0x96, 0xa6, 0x52, 0xb4, 0x23, 0xf7, 0xa0, 0x31, - 0x99, 0xc9, 0xb7, 0xb1, 0x3f, 0x87, 0xb0, 0x8e, 0x71, 0xd7, 0xf1, 0xf4, 0x79, 0x86, 0xe3, 0x5d, + 0x11, 0x15, 0xe1, 0xa0, 0xb4, 0x85, 0xa8, 0x12, 0x52, 0x5d, 0xc7, 0xa5, 0x69, 0x1a, 0xc7, 0x5a, + 0x3b, 0x82, 0xf2, 0xe7, 0x74, 0xf1, 0x6d, 0xd2, 0x53, 0xce, 0xb7, 0xee, 0xee, 0x26, 0x92, 0x79, + 0x05, 0x7e, 0x00, 0x7f, 0x79, 0x03, 0x1e, 0x81, 0x27, 0xe0, 0x39, 0xe0, 0x49, 0xd0, 0xce, 0xde, + 0xd9, 0x67, 0xbb, 0x2d, 0x29, 0xca, 0xbf, 0x99, 0xdd, 0x99, 0xb9, 0x99, 0x6f, 0xe6, 0x9b, 0xb5, + 0xa1, 0xa1, 0xf8, 0x84, 0x87, 0x81, 0x0a, 0x5a, 0x13, 0xc1, 0x15, 0x27, 0x95, 0x4c, 0x6f, 0xee, + 0x42, 0xe5, 0x90, 0x4d, 0x69, 0x90, 0x9c, 0x33, 0xb2, 0x09, 0x25, 0xa9, 0x02, 0xa1, 0x5c, 0x6b, + 0xcb, 0xda, 0xae, 0x53, 0xa3, 0x10, 0x07, 0x8a, 0x2c, 0x09, 0xdd, 0x02, 0x9e, 0x69, 0xb1, 0xf9, + 0x00, 0x6a, 0xc3, 0xe0, 0x34, 0x66, 0xaa, 0x1d, 0x47, 0x81, 0x24, 0x04, 0xec, 0x11, 0x8b, 0x63, + 0xf4, 0xaa, 0x52, 0x94, 0xb5, 0xd3, 0x65, 0x64, 0x9c, 0xd6, 0xa9, 0x16, 0x9b, 0x7f, 0xda, 0x50, + 0x36, 0x5e, 0xe4, 0x0b, 0x28, 0x05, 0xda, 0x13, 0x3d, 0x6a, 0xbb, 0xb7, 0x5b, 0xb3, 0xec, 0x72, + 0x61, 0xa9, 0xb1, 0x21, 0x1e, 0x54, 0x5e, 0x73, 0xa9, 0x92, 0x60, 0xcc, 0x30, 0x5c, 0x95, 0xce, + 0x74, 0xb2, 0x07, 0x95, 0x09, 0x17, 0xca, 0x1f, 0x07, 0x13, 0xd7, 0xde, 0x2a, 0x6e, 0xd7, 0x76, + 0xef, 0x2e, 0xc7, 0x6a, 0xf5, 0xb9, 0x50, 0x47, 0xc1, 0xa4, 0x9b, 0x28, 0x31, 0xa5, 0x6b, 0x13, + 0xa3, 0xe9, 0xa8, 0x17, 0x6c, 0x2a, 0x27, 0xc1, 0x88, 0xb9, 0x25, 0x13, 0x35, 0xd3, 0x11, 0x86, + 0xd7, 0x81, 0x08, 0xdd, 0x32, 0x5e, 0x18, 0x85, 0xec, 0x40, 0xf5, 0x82, 0x4d, 0x7d, 0xa1, 0x91, + 0x72, 0xd7, 0x30, 0x71, 0x32, 0xff, 0x58, 0x86, 0x21, 0x86, 0x31, 0x68, 0x6e, 0x83, 0xad, 0xa6, + 0x13, 0xe6, 0x56, 0xb6, 0xac, 0xed, 0xc6, 0xee, 0xe6, 0x72, 0x62, 0xc3, 0xe9, 0x84, 0x51, 0xb4, + 0x20, 0xdb, 0xe0, 0x84, 0xa7, 0xbe, 0xae, 0xc8, 0xe7, 0x57, 0x4c, 0x88, 0x28, 0x64, 0x6e, 0x15, + 0xbf, 0xdd, 0x08, 0x4f, 0x7b, 0xc1, 0x98, 0x1d, 0xa7, 0xa7, 0xa4, 0x05, 0xb6, 0x0a, 0xce, 0xa5, + 0x0b, 0x58, 0xac, 0xb7, 0x52, 0xec, 0x30, 0x38, 0x97, 0xa6, 0x52, 0xb4, 0x23, 0xf7, 0xa0, 0x31, + 0x9e, 0xca, 0x37, 0xb1, 0x3f, 0x83, 0xb0, 0x8e, 0x71, 0xd7, 0xf1, 0xf4, 0x79, 0x86, 0xe3, 0x5d, 0x00, 0x63, 0xa6, 0xe1, 0x71, 0xd7, 0xb7, 0xac, 0xed, 0x12, 0xad, 0xe2, 0x89, 0x46, 0xcf, 0x7b, - 0x0c, 0xf5, 0x3c, 0x8a, 0xba, 0xb9, 0x67, 0x6c, 0x96, 0xf6, 0x5b, 0x8b, 0x1a, 0xb2, 0x8b, 0x20, - 0x3e, 0x37, 0x1d, 0x2a, 0x51, 0xa3, 0x3c, 0x2e, 0xec, 0x59, 0xde, 0x57, 0x50, 0x9d, 0x27, 0xf5, + 0x0c, 0xf5, 0x3c, 0x8a, 0xba, 0xb9, 0x17, 0x6c, 0x9a, 0xf6, 0x5b, 0x8b, 0x1a, 0xb2, 0xab, 0x20, + 0xbe, 0x34, 0x1d, 0x2a, 0x51, 0xa3, 0x3c, 0x2e, 0xec, 0x59, 0xde, 0x37, 0x50, 0x9d, 0x25, 0xf5, 0x5f, 0x8e, 0xd5, 0x9c, 0xe3, 0x0b, 0xbb, 0x52, 0x74, 0xec, 0x17, 0x76, 0xa5, 0xe6, 0xd4, 0x9b, - 0xbf, 0x96, 0xa1, 0x34, 0xc0, 0x2e, 0xec, 0x41, 0x7d, 0x12, 0x48, 0xc5, 0x84, 0x7f, 0x8d, 0x09, - 0xaa, 0x19, 0x53, 0x33, 0xa5, 0x4b, 0xfd, 0x2b, 0x5c, 0xa3, 0x7f, 0x5f, 0x43, 0x5d, 0x32, 0x71, - 0xc1, 0x42, 0x5f, 0x37, 0x49, 0xba, 0xc5, 0x55, 0xcc, 0x31, 0xa3, 0xd6, 0x00, 0x6d, 0xb0, 0x9b, - 0x35, 0x39, 0x97, 0x25, 0x79, 0x02, 0xeb, 0x92, 0x9f, 0x8b, 0x11, 0xf3, 0x71, 0x7e, 0x64, 0x3a, - 0xa0, 0x1f, 0x5d, 0xf2, 0x47, 0x23, 0x94, 0x69, 0x5d, 0x2e, 0x14, 0x49, 0x9e, 0xc1, 0x86, 0xc2, - 0x6a, 0xfc, 0x11, 0x4f, 0x94, 0xe0, 0xb1, 0x74, 0xcb, 0xab, 0x43, 0x6e, 0x62, 0x98, 0xa2, 0x3b, - 0xc6, 0x8a, 0x36, 0x54, 0x5e, 0x95, 0xe4, 0x3e, 0xdc, 0x8a, 0xa4, 0x9f, 0xc2, 0xa6, 0x53, 0x8c, - 0x92, 0x31, 0x4e, 0x70, 0x85, 0x6e, 0x44, 0xf2, 0x10, 0xcf, 0x07, 0xe6, 0xd8, 0x7b, 0x0d, 0xb0, - 0x28, 0x88, 0x3c, 0x82, 0x5a, 0x9a, 0x01, 0x4e, 0xb2, 0x75, 0xc5, 0x24, 0x83, 0x9a, 0xcb, 0xba, - 0xa9, 0x7a, 0x09, 0x48, 0xb7, 0xb0, 0x55, 0xd4, 0x4d, 0x45, 0xc5, 0xfb, 0xcd, 0x82, 0x5a, 0xae, - 0xd8, 0x6c, 0x45, 0x58, 0xf3, 0x15, 0xb1, 0x44, 0xca, 0xc2, 0xbb, 0x48, 0x59, 0x7c, 0x27, 0x29, - 0xed, 0x6b, 0x34, 0xf5, 0x0e, 0x94, 0x31, 0x51, 0xe9, 0x96, 0x30, 0xb7, 0x54, 0xf3, 0x7e, 0xb7, - 0x60, 0x7d, 0x09, 0xc5, 0x1b, 0xad, 0x9d, 0x7c, 0x0e, 0xe4, 0x24, 0x0e, 0x46, 0x67, 0x71, 0x24, - 0x95, 0x1e, 0x28, 0x93, 0x82, 0x8d, 0x26, 0xb7, 0x72, 0x37, 0x18, 0x54, 0xea, 0x2c, 0x4f, 0x05, - 0xff, 0x91, 0x25, 0xb8, 0x9b, 0x2a, 0x34, 0xd5, 0xe6, 0x9c, 0x28, 0x39, 0xe5, 0xe6, 0x9f, 0x05, - 0xdc, 0xdc, 0x06, 0x9d, 0x2f, 0x60, 0x13, 0x01, 0x89, 0x92, 0xb1, 0x3f, 0xe2, 0xf1, 0xf9, 0x24, - 0xc1, 0x75, 0x92, 0x32, 0x8d, 0x64, 0x77, 0x1d, 0xbc, 0xd2, 0x1b, 0x85, 0xbc, 0xb8, 0xec, 0x81, - 0x75, 0x16, 0xb0, 0x4e, 0x77, 0x09, 0x44, 0xfc, 0xc6, 0x81, 0x99, 0xf1, 0x95, 0x58, 0x58, 0xf3, - 0x93, 0x39, 0x53, 0x4e, 0x05, 0x9f, 0xc8, 0xcb, 0xab, 0x38, 0x8b, 0x91, 0x92, 0xe5, 0x99, 0xe0, - 0x93, 0x8c, 0x2c, 0x5a, 0x96, 0xde, 0x79, 0x36, 0x76, 0x5a, 0xbd, 0x59, 0xe8, 0xf3, 0x43, 0x55, - 0x5c, 0x1e, 0x2a, 0x83, 0x67, 0xf3, 0x27, 0x0b, 0x1c, 0xc3, 0x3f, 0x36, 0x8d, 0xa3, 0x51, 0xa0, - 0x22, 0x9e, 0x90, 0x47, 0x50, 0x4a, 0x78, 0xc8, 0xf4, 0x86, 0xd1, 0xc5, 0x7c, 0xb2, 0x42, 0xb9, + 0xbf, 0x95, 0xa1, 0x34, 0xc0, 0x2e, 0xec, 0x41, 0x7d, 0x1c, 0x48, 0xc5, 0x84, 0x7f, 0x8d, 0x09, + 0xaa, 0x19, 0x53, 0x33, 0xa5, 0x0b, 0xfd, 0x2b, 0x5c, 0xa3, 0x7f, 0xdf, 0x42, 0x5d, 0x32, 0x71, + 0xc5, 0x42, 0x5f, 0x37, 0x49, 0xba, 0xc5, 0x65, 0xcc, 0x31, 0xa3, 0xd6, 0x00, 0x6d, 0xb0, 0x9b, + 0x35, 0x39, 0x93, 0x25, 0x79, 0x02, 0xeb, 0x92, 0x5f, 0x8a, 0x11, 0xf3, 0x71, 0x7e, 0x64, 0x3a, + 0xa0, 0x9f, 0xac, 0xf8, 0xa3, 0x11, 0xca, 0xb4, 0x2e, 0xe7, 0x8a, 0x24, 0xcf, 0x60, 0x43, 0x61, + 0x35, 0xfe, 0x88, 0x27, 0x4a, 0xf0, 0x58, 0xba, 0xe5, 0xe5, 0x21, 0x37, 0x31, 0x4c, 0xd1, 0x1d, + 0x63, 0x45, 0x1b, 0x2a, 0xaf, 0x4a, 0x72, 0x1f, 0x6e, 0x45, 0xd2, 0x4f, 0x61, 0xd3, 0x29, 0x46, + 0xc9, 0x39, 0x4e, 0x70, 0x85, 0x6e, 0x44, 0xf2, 0x08, 0xcf, 0x07, 0xe6, 0xd8, 0x7b, 0x05, 0x30, + 0x2f, 0x88, 0x3c, 0x82, 0x5a, 0x9a, 0x01, 0x4e, 0xb2, 0xf5, 0x9e, 0x49, 0x06, 0x35, 0x93, 0x75, + 0x53, 0xf5, 0x12, 0x90, 0x6e, 0x61, 0xab, 0xa8, 0x9b, 0x8a, 0x8a, 0xf7, 0xbb, 0x05, 0xb5, 0x5c, + 0xb1, 0xd9, 0x8a, 0xb0, 0x66, 0x2b, 0x62, 0x81, 0x94, 0x85, 0x77, 0x91, 0xb2, 0xf8, 0x4e, 0x52, + 0xda, 0xd7, 0x68, 0xea, 0x1d, 0x28, 0x63, 0xa2, 0xd2, 0x2d, 0x61, 0x6e, 0xa9, 0xe6, 0xfd, 0x61, + 0xc1, 0xfa, 0x02, 0x8a, 0x37, 0x5a, 0x3b, 0xf9, 0x12, 0xc8, 0x69, 0x1c, 0x8c, 0x2e, 0xe2, 0x48, + 0x2a, 0x3d, 0x50, 0x26, 0x05, 0x1b, 0x4d, 0x6e, 0xe5, 0x6e, 0x30, 0xa8, 0xd4, 0x59, 0x9e, 0x09, + 0xfe, 0x13, 0x4b, 0x70, 0x37, 0x55, 0x68, 0xaa, 0xcd, 0x38, 0x51, 0x72, 0xca, 0xcd, 0xbf, 0x0a, + 0xb8, 0xb9, 0x0d, 0x3a, 0x5f, 0xc1, 0x26, 0x02, 0x12, 0x25, 0xe7, 0xfe, 0x88, 0xc7, 0x97, 0xe3, + 0x04, 0xd7, 0x49, 0xca, 0x34, 0x92, 0xdd, 0x75, 0xf0, 0x4a, 0x6f, 0x14, 0xf2, 0x62, 0xd5, 0x03, + 0xeb, 0x2c, 0x60, 0x9d, 0xee, 0x02, 0x88, 0xf8, 0x8d, 0x03, 0x33, 0xe3, 0x4b, 0xb1, 0xb0, 0xe6, + 0x27, 0x33, 0xa6, 0x9c, 0x09, 0x3e, 0x96, 0xab, 0xab, 0x38, 0x8b, 0x91, 0x92, 0xe5, 0x99, 0xe0, + 0xe3, 0x8c, 0x2c, 0x5a, 0x96, 0xde, 0x65, 0x36, 0x76, 0x5a, 0xbd, 0x59, 0xe8, 0xf3, 0x43, 0x55, + 0x5c, 0x1c, 0x2a, 0x83, 0x67, 0xf3, 0x67, 0x0b, 0x1c, 0xc3, 0x3f, 0x36, 0x89, 0xa3, 0x51, 0xa0, + 0x22, 0x9e, 0x90, 0x47, 0x50, 0x4a, 0x78, 0xc8, 0xf4, 0x86, 0xd1, 0xc5, 0x7c, 0xb6, 0x44, 0xb9, 0x9c, 0x69, 0xab, 0xc7, 0x43, 0x46, 0x8d, 0xb5, 0xf7, 0x04, 0x6c, 0xad, 0xea, 0x3d, 0x95, 0x96, - 0x70, 0x9d, 0x3d, 0xa5, 0x16, 0x4a, 0xf3, 0x18, 0x1a, 0xe9, 0x17, 0x4e, 0x99, 0x60, 0xc9, 0x88, - 0xe9, 0xf7, 0x35, 0xd7, 0x4c, 0x94, 0xdf, 0x7b, 0x9b, 0x35, 0x7f, 0xb6, 0x80, 0x60, 0xdc, 0xe5, - 0x29, 0xbf, 0x89, 0xd8, 0xe4, 0x21, 0xdc, 0x79, 0x7b, 0xce, 0xc4, 0xcc, 0x2c, 0x97, 0x11, 0xf3, + 0x70, 0x9d, 0x3d, 0xa5, 0xe6, 0x4a, 0xf3, 0x04, 0x1a, 0xe9, 0x17, 0xce, 0x98, 0x60, 0xc9, 0x88, + 0xe9, 0xf7, 0x35, 0xd7, 0x4c, 0x94, 0x3f, 0x78, 0x9b, 0x35, 0x7f, 0xb1, 0x80, 0x60, 0xdc, 0xc5, + 0x29, 0xbf, 0x89, 0xd8, 0xe4, 0x21, 0xdc, 0x79, 0x73, 0xc9, 0xc4, 0xd4, 0x2c, 0x97, 0x11, 0xf3, 0xc3, 0x48, 0xea, 0xaf, 0x18, 0xb2, 0x56, 0xe8, 0x26, 0xde, 0x0e, 0xcc, 0xe5, 0x7e, 0x7a, 0xd7, - 0xfc, 0xdb, 0x86, 0xda, 0x40, 0x5c, 0xcc, 0x67, 0xf8, 0x1b, 0x80, 0x69, 0x20, 0x54, 0xa4, 0x31, - 0xcd, 0x60, 0xff, 0x34, 0x07, 0xfb, 0xc2, 0x74, 0x3e, 0x4f, 0xfd, 0xcc, 0x9e, 0xe6, 0x5c, 0xdf, - 0x49, 0x86, 0xc2, 0x7b, 0x93, 0xa1, 0xf8, 0x3f, 0xc8, 0xd0, 0x86, 0x5a, 0x8e, 0x0c, 0x29, 0x17, - 0xb6, 0xfe, 0xbd, 0x8e, 0x1c, 0x1d, 0x60, 0x41, 0x07, 0xef, 0x2f, 0x0b, 0x6e, 0x5d, 0x2a, 0x51, - 0xb3, 0x22, 0xf7, 0x1e, 0x5d, 0xcd, 0x8a, 0xc5, 0x43, 0x44, 0x3a, 0xe0, 0x60, 0x96, 0xbe, 0xc8, - 0x06, 0xca, 0x10, 0xa4, 0x96, 0xaf, 0x6b, 0x79, 0xe2, 0xe8, 0x86, 0x5c, 0xd2, 0x25, 0xe9, 0xc3, - 0x6d, 0x13, 0x64, 0xf5, 0x41, 0x32, 0x8f, 0xe2, 0xc7, 0x2b, 0x91, 0x96, 0xdf, 0xa3, 0x0f, 0xe5, - 0xa5, 0x33, 0xe9, 0xf9, 0x37, 0xc1, 0xf8, 0x2b, 0x1e, 0x8c, 0x74, 0x4b, 0xfe, 0x00, 0x95, 0x0e, - 0x8b, 0xe3, 0x83, 0xe4, 0x94, 0xeb, 0x1f, 0x43, 0x88, 0x8b, 0xf0, 0x83, 0x30, 0x14, 0x4c, 0xca, - 0x74, 0xea, 0xd7, 0xcd, 0x69, 0xdb, 0x1c, 0x6a, 0x4a, 0x08, 0xce, 0x55, 0x1a, 0x10, 0x65, 0xbd, - 0x90, 0x05, 0x1b, 0x47, 0x3c, 0x49, 0x57, 0x48, 0xaa, 0xdd, 0xdf, 0x85, 0xc6, 0xf2, 0x48, 0x90, - 0x2a, 0x94, 0x8e, 0x7b, 0x83, 0xee, 0xd0, 0xf9, 0x80, 0x00, 0x94, 0x8f, 0x0f, 0x7a, 0xc3, 0x2f, - 0x1f, 0x3a, 0x96, 0x3e, 0x7e, 0xfa, 0x7a, 0xd8, 0x1d, 0x38, 0x85, 0xfb, 0xbf, 0x58, 0x00, 0x8b, - 0x7a, 0x48, 0x0d, 0xd6, 0x8e, 0x7b, 0x2f, 0x7b, 0x47, 0xdf, 0xf6, 0x8c, 0xcb, 0x61, 0x7b, 0x30, - 0xec, 0x52, 0xc7, 0xd2, 0x17, 0xb4, 0xdb, 0x7f, 0x75, 0xd0, 0x69, 0x3b, 0x05, 0x7d, 0x41, 0xf7, - 0x8f, 0x7a, 0xaf, 0x5e, 0x3b, 0x45, 0x8c, 0xd5, 0x1e, 0x76, 0x9e, 0x1b, 0x71, 0xd0, 0x6f, 0xd3, - 0xae, 0x63, 0x13, 0x07, 0xea, 0xdd, 0xef, 0xfa, 0x5d, 0x7a, 0x70, 0xd8, 0xed, 0x0d, 0xdb, 0xaf, - 0x9c, 0x92, 0xf6, 0x79, 0xda, 0xee, 0xbc, 0x3c, 0xee, 0x3b, 0x65, 0x13, 0x6c, 0x30, 0x3c, 0xa2, - 0x5d, 0x67, 0x4d, 0x2b, 0xfb, 0xb4, 0x7d, 0xd0, 0xeb, 0xee, 0x3b, 0x15, 0xaf, 0xe0, 0x58, 0x4f, - 0xf7, 0x60, 0x23, 0xe2, 0xad, 0x8b, 0x48, 0x31, 0x29, 0xcd, 0x3f, 0x84, 0xef, 0xef, 0xa5, 0x5a, - 0xc4, 0x77, 0x8c, 0xb4, 0x33, 0xe6, 0x3b, 0x17, 0x6a, 0x07, 0x6f, 0x77, 0xb2, 0xc6, 0x9c, 0x94, - 0x51, 0x7f, 0xf0, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd2, 0x59, 0xb6, 0xea, 0x61, 0x0c, 0x00, - 0x00, + 0xfc, 0xc7, 0x86, 0xda, 0x40, 0x5c, 0xcd, 0x66, 0xf8, 0x3b, 0x80, 0x49, 0x20, 0x54, 0xa4, 0x31, + 0xcd, 0x60, 0xff, 0x3c, 0x07, 0xfb, 0xdc, 0x74, 0x36, 0x4f, 0xfd, 0xcc, 0x9e, 0xe6, 0x5c, 0xdf, + 0x49, 0x86, 0xc2, 0x07, 0x93, 0xa1, 0xf8, 0x3f, 0xc8, 0xd0, 0x86, 0x5a, 0x8e, 0x0c, 0x29, 0x17, + 0xb6, 0xde, 0x5e, 0x47, 0x8e, 0x0e, 0x30, 0xa7, 0x83, 0xf7, 0xb7, 0x05, 0xb7, 0x56, 0x4a, 0xd4, + 0xac, 0xc8, 0xbd, 0x47, 0xef, 0x67, 0xc5, 0xfc, 0x21, 0x22, 0x1d, 0x70, 0x30, 0x4b, 0x5f, 0x64, + 0x03, 0x65, 0x08, 0x52, 0xcb, 0xd7, 0xb5, 0x38, 0x71, 0x74, 0x43, 0x2e, 0xe8, 0x92, 0xf4, 0xe1, + 0xb6, 0x09, 0xb2, 0xfc, 0x20, 0x99, 0x47, 0xf1, 0xd3, 0xa5, 0x48, 0x8b, 0xef, 0xd1, 0xc7, 0x72, + 0xe5, 0x4c, 0x7a, 0xfe, 0x4d, 0x30, 0xfe, 0x3d, 0x0f, 0x46, 0xba, 0x25, 0x0f, 0xa1, 0xd2, 0x61, + 0x71, 0x7c, 0x90, 0x9c, 0x71, 0xfd, 0x63, 0x08, 0x71, 0x11, 0x7e, 0x10, 0x86, 0x82, 0x49, 0x99, + 0x4e, 0xfd, 0xba, 0x39, 0x6d, 0x9b, 0x43, 0x4d, 0x09, 0xc1, 0xb9, 0x4a, 0x03, 0xa2, 0x9c, 0x2e, + 0x8a, 0x26, 0x80, 0x0e, 0x26, 0xcd, 0x0f, 0x8a, 0xb7, 0xae, 0x9b, 0xfb, 0xbb, 0xd0, 0x58, 0x1c, + 0x12, 0x52, 0x85, 0xd2, 0x49, 0x6f, 0xd0, 0x1d, 0x3a, 0x1f, 0x11, 0x80, 0xf2, 0xc9, 0x41, 0x6f, + 0xf8, 0xf5, 0x43, 0xc7, 0xd2, 0xc7, 0x4f, 0x5f, 0x0d, 0xbb, 0x03, 0xa7, 0x70, 0xff, 0x57, 0x0b, + 0x60, 0x5e, 0x21, 0xa9, 0xc1, 0xda, 0x49, 0xef, 0xb0, 0x77, 0xfc, 0x7d, 0xcf, 0xb8, 0x1c, 0xb5, + 0x07, 0xc3, 0x2e, 0x75, 0x2c, 0x7d, 0x41, 0xbb, 0xfd, 0x97, 0x07, 0x9d, 0xb6, 0x53, 0xd0, 0x17, + 0x74, 0xff, 0xb8, 0xf7, 0xf2, 0x95, 0x53, 0xc4, 0x58, 0xed, 0x61, 0xe7, 0xb9, 0x11, 0x07, 0xfd, + 0x36, 0xed, 0x3a, 0x36, 0x71, 0xa0, 0xde, 0xfd, 0xa1, 0xdf, 0xa5, 0x07, 0x47, 0xdd, 0xde, 0xb0, + 0xfd, 0xd2, 0x29, 0x69, 0x9f, 0xa7, 0xed, 0xce, 0xe1, 0x49, 0xdf, 0x29, 0x9b, 0x60, 0x83, 0xe1, + 0x31, 0xed, 0x3a, 0x6b, 0x5a, 0xd9, 0xa7, 0xed, 0x83, 0x5e, 0x77, 0xdf, 0xa9, 0x78, 0x05, 0xc7, + 0x7a, 0xba, 0x07, 0x1b, 0x11, 0x6f, 0x5d, 0x45, 0x8a, 0x49, 0x69, 0xfe, 0x33, 0xfc, 0x78, 0x2f, + 0xd5, 0x22, 0xbe, 0x63, 0xa4, 0x9d, 0x73, 0xbe, 0x73, 0xa5, 0x76, 0xf0, 0x76, 0x27, 0x6b, 0xd5, + 0x69, 0x19, 0xf5, 0x07, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x89, 0xf4, 0xf5, 0x28, 0x73, 0x0c, + 0x00, 0x00, } diff --git a/go/vt/topo/cells_aliases.go b/go/vt/topo/cells_aliases.go new file mode 100644 index 00000000000..6488a095aba --- /dev/null +++ b/go/vt/topo/cells_aliases.go @@ -0,0 +1,174 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package topo + +import ( + "fmt" + "path" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +// This file provides the utility methods to save / retrieve CellsAliases +// in the topology server. +// +// CellsAliases records are not meant to be changed while the system is +// running. In a running system, a CellsAlias can be added, and +// topology server implementations should be able to read them to +// access the cells upon demand. + +func pathForCellsAlias(alias string) string { + return path.Join(CellsAliasesPath, alias, CellsAliasFile) +} + +// GetCellsAliases returns the names of the existing cells. They are +// sorted by name. +func (ts *Server) GetCellsAliases(ctx context.Context, strongRead bool) (ret map[string]*topodatapb.CellsAlias, err error) { + conn := ts.globalCell + if !strongRead { + conn = ts.globalReadOnlyCell + } + + entries, err := ts.globalCell.ListDir(ctx, CellsAliasesPath, false /*full*/) + switch { + case IsErrType(err, NoNode): + return nil, nil + case err == nil: + aliases := DirEntriesToStringArray(entries) + ret = make(map[string]*topodatapb.CellsAlias, len(aliases)) + for _, alias := range aliases { + aliasPath := pathForCellsAlias(alias) + contents, _, err := conn.Get(ctx, aliasPath) + if err != nil { + return nil, err + } + + // Unpack the contents. + cellsAlias := &topodatapb.CellsAlias{} + if err := proto.Unmarshal(contents, cellsAlias); err != nil { + return nil, err + } + + ret[alias] = cellsAlias + } + return ret, nil + default: + return nil, err + } +} + +// DeleteCellsAlias deletes the specified CellsAlias +func (ts *Server) DeleteCellsAlias(ctx context.Context, alias string) error { + ts.clearCellAliasesCache() + + filePath := pathForCellsAlias(alias) + return ts.globalCell.Delete(ctx, filePath, nil) +} + +// CreateCellsAlias creates a new CellInfo with the provided content. +func (ts *Server) CreateCellsAlias(ctx context.Context, alias string, cellsAlias *topodatapb.CellsAlias) error { + currentAliases, err := ts.GetCellsAliases(ctx, true) + if err != nil { + return err + } + + if overlappingAliases(currentAliases, cellsAlias) { + return fmt.Errorf("unsupported: you can't over overlapping aliases. Cells alias: %v, has an overlap with existent aliases", cellsAlias) + + } + + ts.clearCellAliasesCache() + + // Pack the content. + contents, err := proto.Marshal(cellsAlias) + if err != nil { + return err + } + + // Save it. + filePath := pathForCellsAlias(alias) + _, err = ts.globalCell.Create(ctx, filePath, contents) + return err +} + +// UpdateCellsAlias updates cells for a given alias +func (ts *Server) UpdateCellsAlias(ctx context.Context, alias string, update func(*topodatapb.CellsAlias) error) error { + ts.clearCellAliasesCache() + + filePath := pathForCellsAlias(alias) + for { + ca := &topodatapb.CellsAlias{} + + // Read the file, unpack the contents. + contents, version, err := ts.globalCell.Get(ctx, filePath) + switch { + case err == nil: + if err := proto.Unmarshal(contents, ca); err != nil { + return err + } + case IsErrType(err, NoNode): + // Nothing to do. + default: + return err + } + + // Call update method. + if err = update(ca); err != nil { + if IsErrType(err, NoUpdateNeeded) { + return nil + } + return err + } + + currentAliases, err := ts.GetCellsAliases(ctx, true) + if err != nil { + return err + } + + if overlappingAliases(currentAliases, ca) { + return fmt.Errorf("unsupported: you can't over overlapping aliases. Cells alias: %v, has an overlap with existent aliases", ca) + + } + + // Pack and save. + contents, err = proto.Marshal(ca) + if err != nil { + return err + } + if _, err = ts.globalCell.Update(ctx, filePath, contents, version); !IsErrType(err, BadVersion) { + // This includes the 'err=nil' case. + return err + } + } +} + +func overlappingAliases(currentAliases map[string]*topodatapb.CellsAlias, newAlias *topodatapb.CellsAlias) bool { + for _, cellsAlias := range currentAliases { + for _, cell := range cellsAlias.Cells { + for _, newCell := range newAlias.Cells { + if cell == newCell { + return true + } + } + } + + } + return false +} diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index cf6b86acc36..5b390e8673b 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -66,6 +66,7 @@ const ( // Filenames for all object types. const ( CellInfoFile = "CellInfo" + CellsAliasFile = "CellsAlias" KeyspaceFile = "Keyspace" ShardFile = "Shard" VSchemaFile = "VSchema" @@ -77,10 +78,11 @@ const ( // Path for all object types. const ( - CellsPath = "cells" - KeyspacesPath = "keyspaces" - ShardsPath = "shards" - TabletsPath = "tablets" + CellsPath = "cells" + CellsAliasesPath = "cells_aliases" + KeyspacesPath = "keyspaces" + ShardsPath = "shards" + TabletsPath = "tablets" ) // Factory is a factory interface to create Conn objects. @@ -134,10 +136,10 @@ type Server struct { cells map[string]Conn } -type cellsToRegionsMap struct { +type cellsToAliasesMap struct { mu sync.Mutex - // cellsToRegions contains all cell->region mappings - cellsToRegions map[string]string + // cellsToAliases contains all cell->alias mappings + cellsToAliases map[string]string } var ( @@ -155,8 +157,8 @@ var ( // factories has the factories for the Conn objects. factories = make(map[string]Factory) - regions = cellsToRegionsMap{ - cellsToRegions: make(map[string]string), + cellsAliases = cellsToAliasesMap{ + cellsToAliases: make(map[string]string), } ) @@ -271,32 +273,34 @@ func (ts *Server) ConnForCell(ctx context.Context, cell string) (Conn, error) { } } -// GetRegionByCell returns the region group this `cell` belongs to, if there's none, it returns the `cell` as region. -func GetRegionByCell(ctx context.Context, ts *Server, cell string) string { - regions.mu.Lock() - defer regions.mu.Unlock() - if region, ok := regions.cellsToRegions[cell]; ok { +// GetAliasByCell returns the alias group this `cell` belongs to, if there's none, it returns the `cell` as alias. +func GetAliasByCell(ctx context.Context, ts *Server, cell string) string { + cellsAliases.mu.Lock() + defer cellsAliases.mu.Unlock() + if region, ok := cellsAliases.cellsToAliases[cell]; ok { return region } if ts != nil { - // lazily get the region from cell info if `regions.ts` is available - info, err := ts.GetCellInfo(ctx, cell, false) - if err == nil && info.Region != "" { - regions.cellsToRegions[cell] = info.Region - return info.Region + // lazily get the region from cell info if `aliases` are available + cellAliases, err := ts.GetCellsAliases(ctx, false) + if err != nil { + // for backward compatibility + return cell + } + + for alias, cellsAlias := range cellAliases { + for _, cellAlias := range cellsAlias.Cells { + if cellAlias == cell { + cellsAliases.cellsToAliases[cell] = alias + return alias + } + } } } - // for backward compatability + // for backward compatibility return cell } -// UpdateCellsToRegionsForTests overwrites the global map built by topo server init, and is meant for testing purpose only. -func UpdateCellsToRegionsForTests(cellsToRegions map[string]string) { - regions.mu.Lock() - defer regions.mu.Unlock() - regions.cellsToRegions = cellsToRegions -} - // Close will close all connections to underlying topo Server. // It will nil all member variables, so any further access will panic. func (ts *Server) Close() { @@ -313,3 +317,9 @@ func (ts *Server) Close() { } ts.cells = make(map[string]Conn) } + +func (ts *Server) clearCellAliasesCache() { + cellsAliases.mu.Lock() + defer cellsAliases.mu.Unlock() + cellsAliases.cellsToAliases = make(map[string]string) +} diff --git a/go/vt/topo/topotests/cells_aliases_test.go b/go/vt/topo/topotests/cells_aliases_test.go new file mode 100644 index 00000000000..110890d2955 --- /dev/null +++ b/go/vt/topo/topotests/cells_aliases_test.go @@ -0,0 +1,151 @@ +/* +Copyright 2017 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package topotests + +import ( + "reflect" + "sort" + "testing" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/vt/topo/memorytopo" + + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +// This file tests the CellsAliases part of the topo.Server API. + +func TestCellsAliases(t *testing.T) { + // Create an alias + + cell := "cell1" + ctx := context.Background() + ts := memorytopo.NewServer(cell) + + if err := ts.CreateCellsAlias(ctx, "alias", &topodatapb.CellsAlias{Cells: []string{"cell1", "cell2"}}); err != nil { + t.Fatalf("CreateCellsAlias failed: %v", err) + } + + if err := ts.CreateCellsAlias(ctx, "aliasb", &topodatapb.CellsAlias{Cells: []string{"cell3", "cell4"}}); err != nil { + t.Fatalf("CreateCellsAlias failed: %v", err) + } + + aliases, err := ts.GetCellsAliases(ctx, true /*strongRead*/) + if err != nil { + t.Fatalf("GetCellsAliases failed: %v", err) + } + + var aliasesName []string + for aliasName := range aliases { + aliasesName = append(aliasesName, aliasName) + } + sort.Strings(aliasesName) + + if len(aliasesName) != 2 { + t.Fatalf("Expected to have 2 aliases. Got %v", len(aliasesName)) + + } + + if aliasesName[0] != "alias" { + t.Fatalf("Expected alias name to be alias, got: %v", aliasesName[0]) + } + + if aliasesName[1] != "aliasb" { + t.Fatalf("Expected alias name to be aliasb, got: %v", aliasesName[0]) + } + + want := []string{"cell1", "cell2"} + + if !reflect.DeepEqual(aliases[aliasesName[0]].Cells, want) { + t.Fatalf("Expected alias to be: %v, got %v", want, aliases[aliasesName[0]]) + } + + want = []string{"cell3", "cell4"} + + if !reflect.DeepEqual(aliases[aliasesName[1]].Cells, want) { + t.Fatalf("Expected aliasb to be: %v, got %v", want, aliases[aliasesName[1]]) + } + + // Test update on non-existing object. + + want = []string{"newcell"} + + if err := ts.UpdateCellsAlias(ctx, "newalias", func(ca *topodatapb.CellsAlias) error { + ca.Cells = want + return nil + }); err != nil { + t.Fatalf("UpdateCellsAlias failed: %v", err) + } + + aliases, err = ts.GetCellsAliases(ctx, true /*strongRead*/) + if err != nil { + t.Fatalf("GetCellsAliases failed: %v", err) + } + + if !reflect.DeepEqual(aliases["newalias"].Cells, want) { + t.Fatalf("Expected newalias to be: %v, got %v", want, aliases["newalias"]) + } + + // Test update on existing object. + + want = []string{"newcell2"} + + if err := ts.UpdateCellsAlias(ctx, "newalias", func(ca *topodatapb.CellsAlias) error { + ca.Cells = want + return nil + }); err != nil { + t.Fatalf("UpdateCellsAlias failed: %v", err) + } + + aliases, err = ts.GetCellsAliases(ctx, true /*strongRead*/) + if err != nil { + t.Fatalf("GetCellsAliases failed: %v", err) + } + + if !reflect.DeepEqual(aliases["newalias"].Cells, want) { + t.Fatalf("Expected newalias to be: %v, got %v", want, aliases["newalias"]) + } + + // Test delete alias + + if err := ts.DeleteCellsAlias(ctx, "newalias"); err != nil { + t.Fatalf("UpdateCellsAlias failed: %v", err) + } + + aliases, err = ts.GetCellsAliases(ctx, true /*strongRead*/) + if err != nil { + t.Fatalf("GetCellsAliases failed: %v", err) + } + + if aliases["newalias"] != nil { + t.Fatalf("Expected newalias to be: nil, got %v", aliases["newalias"]) + } + + // Create an alias that adds an overlapping cell is not supported + if err := ts.CreateCellsAlias(ctx, "invalid", &topodatapb.CellsAlias{Cells: []string{"cell1", "cell2"}}); err == nil { + t.Fatal("CreateCellsAlias should fail, got nil") + } + + // Update an alias that adds an overlapping cell is not supported + if err := ts.UpdateCellsAlias(ctx, "aliasb", func(ca *topodatapb.CellsAlias) error { + ca.Cells = []string{"cell1"} + return nil + }); err == nil { + t.Fatalf("UpdateCellsAlias should fail, got nil") + } +} diff --git a/go/vt/topo/zk2topo/zk_conn.go b/go/vt/topo/zk2topo/zk_conn.go index c7539192500..e69980768bd 100644 --- a/go/vt/topo/zk2topo/zk_conn.go +++ b/go/vt/topo/zk2topo/zk_conn.go @@ -57,6 +57,7 @@ var ( certPath = flag.String("topo_zk_tls_cert", "", "the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS") keyPath = flag.String("topo_zk_tls_key", "", "the key to use to connect to the zk topo server, enables TLS") caPath = flag.String("topo_zk_tls_ca", "", "the server ca to use to validate servers when connecting to the zk topo server") + authFile = flag.String("topo_zk_auth_file", "", "auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass") ) // Time returns a time.Time from a ZK int64 milliseconds since Epoch time. @@ -195,6 +196,14 @@ func (c *ZkConn) SetACL(ctx context.Context, path string, aclv []zk.ACL, version }) } +// AddAuth is part of the Conn interface. +func (c *ZkConn) AddAuth(ctx context.Context, scheme string, auth []byte) error { + return c.withRetry(ctx, func(conn *zk.Conn) error { + err := conn.AddAuth(scheme, auth) + return err + }) +} + // Close is part of the Conn interface. func (c *ZkConn) Close() error { c.mu.Lock() @@ -271,10 +280,34 @@ func (c *ZkConn) getConn(ctx context.Context) (*zk.Conn, error) { } c.conn = conn go c.handleSessionEvents(conn, events) + c.maybeAddAuth(ctx) } return c.conn, nil } +// maybeAddAuth calls AddAuth if the `-topo_zk_auth_file` flag was specified +func (c *ZkConn) maybeAddAuth(ctx context.Context) { + if *authFile == "" { + return + } + authInfoBytes, err := ioutil.ReadFile(*authFile) + if err != nil { + log.Errorf("failed to read topo_zk_auth_file: %v", err) + return + } + authInfo := string(authInfoBytes) + authInfoParts := strings.SplitN(authInfo, ":", 2) + if len(authInfoParts) != 2 { + log.Errorf("failed to parse topo_zk_auth_file contents, expected format : but saw: %s", authInfo) + return + } + err = c.conn.AddAuth(authInfoParts[0], []byte(authInfoParts[1])) + if err != nil { + log.Errorf("failed to add auth from topo_zk_auth_file: %v", err) + return + } +} + // handleSessionEvents is processing events from the session channel. // When it detects that the connection is not working any more, it // clears out the connection record. diff --git a/go/vt/topotools/rebuild_keyspace.go b/go/vt/topotools/rebuild_keyspace.go index 513849854ae..fcf4f52d45b 100644 --- a/go/vt/topotools/rebuild_keyspace.go +++ b/go/vt/topotools/rebuild_keyspace.go @@ -71,8 +71,6 @@ func RebuildKeyspaceLocked(ctx context.Context, log logutil.Logger, ts *topo.Ser } // This is safe to rebuild as long there are not srvKeyspaces with tablet controls set. - // TODO: Add this validation. - // Build the list of cells to work on: we get the union // of all the Cells of all the Shards, limited to the provided cells. // diff --git a/go/vt/vtctl/cell_info.go b/go/vt/vtctl/cell_info.go index d47bb6cb134..c1917c8934c 100644 --- a/go/vt/vtctl/cell_info.go +++ b/go/vt/vtctl/cell_info.go @@ -39,13 +39,13 @@ func init() { addCommand(cellsGroupName, command{ "AddCellInfo", commandAddCellInfo, - "[-server_address ] [-root ] [-region ] ", + "[-server_address ] [-root ] ", "Registers a local topology service in a new cell by creating the CellInfo with the provided parameters. The address will be used to connect to the topology service, and we'll put Vitess data starting at the provided root."}) addCommand(cellsGroupName, command{ "UpdateCellInfo", commandUpdateCellInfo, - "[-server_address ] [-root ] [-region ] ", + "[-server_address ] [-root ] ", "Updates the content of a CellInfo with the provided parameters. If a value is empty, it is not updated. The CellInfo will be created if it doesn't exist."}) addCommand(cellsGroupName, command{ @@ -70,7 +70,6 @@ func init() { func commandAddCellInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { serverAddress := subFlags.String("server_address", "", "The address the topology server is using for that cell.") root := subFlags.String("root", "", "The root path the topology server is using for that cell.") - region := subFlags.String("region", "", "The region this cell belongs to.") if err := subFlags.Parse(args); err != nil { return err } @@ -82,14 +81,12 @@ func commandAddCellInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl return wr.TopoServer().CreateCellInfo(ctx, cell, &topodatapb.CellInfo{ ServerAddress: *serverAddress, Root: *root, - Region: *region, }) } func commandUpdateCellInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { serverAddress := subFlags.String("server_address", "", "The address the topology server is using for that cell.") root := subFlags.String("root", "", "The root path the topology server is using for that cell.") - region := subFlags.String("region", "", "The region this cell belongs to.") if err := subFlags.Parse(args); err != nil { return err } @@ -100,8 +97,7 @@ func commandUpdateCellInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags return wr.TopoServer().UpdateCellInfoFields(ctx, cell, func(ci *topodatapb.CellInfo) error { if (*serverAddress == "" || ci.ServerAddress == *serverAddress) && - (*root == "" || ci.Root == *root) && - (*region == "" || ci.Region == *region) { + (*root == "" || ci.Root == *root) { return topo.NewError(topo.NoUpdateNeeded, cell) } if *serverAddress != "" { @@ -110,9 +106,6 @@ func commandUpdateCellInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags if *root != "" { ci.Root = *root } - if *region != "" { - ci.Region = *region - } return nil }) } diff --git a/go/vt/vtctl/cells_aliases.go b/go/vt/vtctl/cells_aliases.go new file mode 100644 index 00000000000..5ef2a78252b --- /dev/null +++ b/go/vt/vtctl/cells_aliases.go @@ -0,0 +1,130 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctl + +import ( + "flag" + "fmt" + "strings" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/vt/wrangler" + + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +// This file contains the CellsAliases command group for vtctl. + +const cellsAliasesGroupName = "CellsAliases" + +func init() { + addCommandGroup(cellsAliasesGroupName) + + addCommand(cellsAliasesGroupName, command{ + "AddCellsAlias", + commandAddCellsAlias, + "[-cells ] ", + "Registers a local topology service in a new cell by creating the CellInfo with the provided parameters. The address will be used to connect to the topology service, and we'll put Vitess data starting at the provided root."}) + + addCommand(cellsAliasesGroupName, command{ + "UpdateCellsAlias", + commandUpdateCellsAlias, + "[-cells ] ", + "Updates the content of a CellInfo with the provided parameters. If a value is empty, it is not updated. The CellInfo will be created if it doesn't exist."}) + + addCommand(cellsAliasesGroupName, command{ + "DeleteCellsAlias", + commandDeleteCellsAlias, + "", + "Deletes the CellsAlias for the provided alias."}) + + addCommand(cellsAliasesGroupName, command{ + "GetCellsAliases", + commandGetCellsAliases, + "", + "Lists all the cells for which we have a CellInfo object, meaning we have a local topology service registered."}) +} + +func commandAddCellsAlias(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { + cellsString := subFlags.String("cells", "", "The address the topology server is using for that cell.") + if err := subFlags.Parse(args); err != nil { + return err + } + if subFlags.NArg() != 1 { + return fmt.Errorf("the argument is required for the AddCellsAlias command") + } + + cells := strings.Split(*cellsString, ",") + for i, cell := range cells { + cells[i] = strings.TrimSpace(cell) + } + + alias := subFlags.Arg(0) + + return wr.TopoServer().CreateCellsAlias(ctx, alias, &topodatapb.CellsAlias{ + Cells: cells, + }) +} + +func commandUpdateCellsAlias(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { + cellsString := subFlags.String("cells", "", "The address the topology server is using for that cell.") + if err := subFlags.Parse(args); err != nil { + return err + } + if subFlags.NArg() != 1 { + return fmt.Errorf("the argument is required for the UpdateCellsAlias command") + } + + cells := strings.Split(*cellsString, ",") + for i, cell := range cells { + cells[i] = strings.TrimSpace(cell) + } + + alias := subFlags.Arg(0) + + return wr.TopoServer().UpdateCellsAlias(ctx, alias, func(ca *topodatapb.CellsAlias) error { + ca.Cells = cells + return nil + }) +} + +func commandDeleteCellsAlias(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { + if err := subFlags.Parse(args); err != nil { + return err + } + if subFlags.NArg() != 1 { + return fmt.Errorf("the argument is required for the DeleteCellsAlias command") + } + alias := subFlags.Arg(0) + + return wr.TopoServer().DeleteCellsAlias(ctx, alias) +} + +func commandGetCellsAliases(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { + if err := subFlags.Parse(args); err != nil { + return err + } + if subFlags.NArg() != 0 { + return fmt.Errorf("GetCellsAliases command takes no parameter") + } + aliases, err := wr.TopoServer().GetCellsAliases(ctx, true /*strongRead*/) + if err != nil { + return err + } + return printJSON(wr.Logger(), aliases) +} diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 0e4df702cef..80fbedc9859 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -2103,7 +2103,7 @@ func commandGetPermissions(ctx context.Context, wr *wrangler.Wrangler, subFlags } p, err := wr.GetPermissions(ctx, tabletAlias) if err == nil { - wr.Logger().Infof("%v", p.String()) // they can contain '%' + printJSON(wr.Logger(), p) } return err } diff --git a/go/vt/vtgate/gateway/discoverygateway_test.go b/go/vt/vtgate/gateway/discoverygateway_test.go index e8dd72cdeb7..1acf96f169b 100644 --- a/go/vt/vtgate/gateway/discoverygateway_test.go +++ b/go/vt/vtgate/gateway/discoverygateway_test.go @@ -25,7 +25,9 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/discovery" + "vitess.io/vitess/go/vt/srvtopo/srvtopotest" "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vterrors" @@ -129,12 +131,6 @@ func TestDiscoveryGatewayGetTablets(t *testing.T) { } func TestShuffleTablets(t *testing.T) { - defer topo.UpdateCellsToRegionsForTests(map[string]string{}) - topo.UpdateCellsToRegionsForTests(map[string]string{ - "cell1": "region1", - "cell2": "region1", - }) - ts1 := discovery.TabletStats{ Key: "t1", Tablet: topo.NewTablet(10, "cell1", "host1"), @@ -237,13 +233,18 @@ func TestDiscoveryGatewayGetAggregateStatsRegion(t *testing.T) { keyspace := "ks" shard := "0" hc := discovery.NewFakeHealthCheck() - dg := createDiscoveryGateway(hc, nil, "local-east", 2).(*discoveryGateway) + ts := memorytopo.NewServer("local-west", "local-east", "remote") + srvTopo := srvtopotest.NewPassthroughSrvTopoServer() + srvTopo.TopoServer = ts + dg := createDiscoveryGateway(hc, srvTopo, "local-east", 2).(*discoveryGateway) - topo.UpdateCellsToRegionsForTests(map[string]string{ - "local-west": "local", - "local-east": "local", - "remote": "remote", - }) + cellsAlias := &topodatapb.CellsAlias{ + Cells: []string{"local-west", "local-east"}, + } + + ts.CreateCellsAlias(context.Background(), "local", cellsAlias) + + defer ts.DeleteCellsAlias(context.Background(), "local") hc.Reset() dg.tsc.ResetForTesting() @@ -308,18 +309,55 @@ func TestDiscoveryGatewayGetAggregateStatsMaster(t *testing.T) { } } +func TestDiscoveryGatewayGetTabletsInRegion(t *testing.T) { + keyspace := "ks" + shard := "0" + hc := discovery.NewFakeHealthCheck() + ts := memorytopo.NewServer("local-west", "local-east", "local", "remote") + srvTopo := srvtopotest.NewPassthroughSrvTopoServer() + srvTopo.TopoServer = ts + + cellsAlias := &topodatapb.CellsAlias{ + Cells: []string{"local-west", "local-east"}, + } + + dg := createDiscoveryGateway(hc, srvTopo, "local-west", 2).(*discoveryGateway) + + ts.CreateCellsAlias(context.Background(), "local", cellsAlias) + + defer ts.DeleteCellsAlias(context.Background(), "local") + + // this is a test + // replica should only use local ones + hc.Reset() + dg.tsc.ResetForTesting() + hc.AddTestTablet("remote", "1.1.1.1", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil) + ep1 := hc.AddTestTablet("local-west", "2.2.2.2", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil).Tablet() + ep2 := hc.AddTestTablet("local-east", "3.3.3.3", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil).Tablet() + tsl := dg.tsc.GetHealthyTabletStats(keyspace, shard, topodatapb.TabletType_REPLICA) + if len(tsl) != 2 || (!topo.TabletEquality(tsl[0].Tablet, ep1) && !topo.TabletEquality(tsl[0].Tablet, ep2)) { + t.Fatalf("want %+v or %+v, got %+v", ep1, ep2, tsl) + } +} func TestDiscoveryGatewayGetTabletsWithRegion(t *testing.T) { keyspace := "ks" shard := "0" hc := discovery.NewFakeHealthCheck() - dg := createDiscoveryGateway(hc, nil, "local", 2).(*discoveryGateway) - topo.UpdateCellsToRegionsForTests(map[string]string{ - "local-west": "local", - "local-east": "local", - "local": "local", - "remote": "remote", - }) + ts := memorytopo.NewServer("local-west", "local-east", "local", "remote") + srvTopo := srvtopotest.NewPassthroughSrvTopoServer() + srvTopo.TopoServer = ts + + cellsAlias := &topodatapb.CellsAlias{ + Cells: []string{"local-west", "local-east"}, + } + + dg := createDiscoveryGateway(hc, srvTopo, "local", 2).(*discoveryGateway) + + ts.CreateCellsAlias(context.Background(), "local", cellsAlias) + + defer ts.DeleteCellsAlias(context.Background(), "local") + // this is a test // replica should only use local ones hc.Reset() dg.tsc.ResetForTesting() @@ -328,7 +366,7 @@ func TestDiscoveryGatewayGetTabletsWithRegion(t *testing.T) { ep2 := hc.AddTestTablet("local-east", "3.3.3.3", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil).Tablet() tsl := dg.tsc.GetHealthyTabletStats(keyspace, shard, topodatapb.TabletType_REPLICA) if len(tsl) != 2 || (!topo.TabletEquality(tsl[0].Tablet, ep1) && !topo.TabletEquality(tsl[0].Tablet, ep2)) { - t.Errorf("want %+v or %+v, got %+v", ep1, ep2, tsl) + t.Fatalf("want %+v or %+v, got %+v", ep1, ep2, tsl) } } @@ -351,7 +389,7 @@ func benchmarkCellsGetAggregateStats(i int, b *testing.B) { cellsToregions[cell] = "local" } - topo.UpdateCellsToRegionsForTests(cellsToregions) + //topo.UpdateCellsToRegionsForTests(cellsToregions) hc.Reset() dg.tsc.ResetForTesting() diff --git a/go/vt/vttablet/tabletmanager/init_tablet.go b/go/vt/vttablet/tabletmanager/init_tablet.go index 715b8744953..ba4ed6e8a53 100644 --- a/go/vt/vttablet/tabletmanager/init_tablet.go +++ b/go/vt/vttablet/tabletmanager/init_tablet.go @@ -131,7 +131,7 @@ func (agent *ActionAgent) InitTablet(port, gRPCPort int32) error { case err == nil: // NOOP case topo.IsErrType(err, topo.NoNode): - err = topotools.RebuildKeyspaceLocked(ctx, logutil.NewConsoleLogger(), agent.TopoServer, *initKeyspace, []string{agent.TabletAlias.Cell}) + err = topotools.RebuildKeyspace(ctx, logutil.NewConsoleLogger(), agent.TopoServer, *initKeyspace, []string{agent.TabletAlias.Cell}) default: return vterrors.Wrap(err, "InitTablet failed to read srvKeyspace") } diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go index 62b1e2c2e2f..e2fa8dd6838 100644 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ b/go/vt/vttablet/tabletmanager/state_change.go @@ -220,7 +220,7 @@ func (agent *ActionAgent) changeCallback(ctx context.Context, oldTablet, newTabl } srvKeyspace, err := agent.TopoServer.GetSrvKeyspace(ctx, newTablet.Alias.Cell, newTablet.Keyspace) if err != nil { - log.Error("Fail to get SrvKeyspace") + log.Errorf("failed to get SrvKeyspace %v with: %v", newTablet.Keyspace, err) } else { for _, partition := range srvKeyspace.GetPartitions() { diff --git a/go/vt/vttablet/tabletserver/messager/engine.go b/go/vt/vttablet/tabletserver/messager/engine.go index b13e81bc8a6..46875d994af 100644 --- a/go/vt/vttablet/tabletserver/messager/engine.go +++ b/go/vt/vttablet/tabletserver/messager/engine.go @@ -130,6 +130,11 @@ func (me *Engine) Subscribe(ctx context.Context, name string, send func(*sqltype // LockDB obtains db locks for all messages that need to // be updated and returns the counterpart unlock function. func (me *Engine) LockDB(newMessages map[string][]*MessageRow, changedMessages map[string][]string) func() { + // Short-circuit to avoid taking any locks if there's nothing to do. + if len(newMessages) == 0 && len(changedMessages) == 0 { + return func() {} + } + // Build the set of affected messages tables. combined := make(map[string]struct{}) for name := range newMessages { @@ -174,6 +179,11 @@ func (me *Engine) LockDB(newMessages map[string][]*MessageRow, changedMessages m // UpdateCaches updates the caches for the committed changes. func (me *Engine) UpdateCaches(newMessages map[string][]*MessageRow, changedMessages map[string][]string) { + // Short-circuit to avoid taking any locks if there's nothing to do. + if len(newMessages) == 0 && len(changedMessages) == 0 { + return + } + me.mu.Lock() defer me.mu.Unlock() now := time.Now().UnixNano() diff --git a/go/vt/vttablet/tabletserver/planbuilder/plan.go b/go/vt/vttablet/tabletserver/planbuilder/plan.go index befeae05e40..0dc8809e4fa 100644 --- a/go/vt/vttablet/tabletserver/planbuilder/plan.go +++ b/go/vt/vttablet/tabletserver/planbuilder/plan.go @@ -137,31 +137,6 @@ func (pt PlanType) MarshalJSON() ([]byte, error) { return json.Marshal(pt.String()) } -// MinRole is the minimum Role required to execute this PlanType. -func (pt PlanType) MinRole() tableacl.Role { - return tableACLRoles[pt] -} - -var tableACLRoles = map[PlanType]tableacl.Role{ - PlanPassSelect: tableacl.READER, - PlanSelectLock: tableacl.READER, - PlanSet: tableacl.READER, - PlanPassDML: tableacl.WRITER, - PlanDMLPK: tableacl.WRITER, - PlanDMLSubquery: tableacl.WRITER, - PlanInsertPK: tableacl.WRITER, - PlanInsertSubquery: tableacl.WRITER, - PlanInsertMessage: tableacl.WRITER, - PlanDDL: tableacl.ADMIN, - PlanSelectStream: tableacl.READER, - PlanOtherRead: tableacl.READER, - PlanOtherAdmin: tableacl.ADMIN, - PlanUpsertPK: tableacl.WRITER, - PlanNextval: tableacl.WRITER, - PlanMessageStream: tableacl.WRITER, - PlanSelectImpossible: tableacl.READER, -} - //_______________________________________________ // ReasonType indicates why a query plan fails to build diff --git a/go/vt/vttablet/tabletserver/query_engine.go b/go/vt/vttablet/tabletserver/query_engine.go index 355a6dea6e4..c173724c851 100644 --- a/go/vt/vttablet/tabletserver/query_engine.go +++ b/go/vt/vttablet/tabletserver/query_engine.go @@ -58,10 +58,9 @@ import ( // and track stats. type TabletPlan struct { *planbuilder.Plan - Fields []*querypb.Field - Rules *rules.Rules - LegacyAuthorized *tableacl.ACLResult - Authorized []*tableacl.ACLResult + Fields []*querypb.Field + Rules *rules.Rules + Authorized []*tableacl.ACLResult mu sync.Mutex QueryCount int64 @@ -344,7 +343,6 @@ func (qe *QueryEngine) GetPlan(ctx context.Context, logStats *tabletenv.LogStats } plan := &TabletPlan{Plan: splan} plan.Rules = qe.queryRuleSources.FilterByPlan(sql, plan.PlanID, plan.TableName().String()) - plan.LegacyAuthorized = tableacl.Authorized(plan.TableName().String(), plan.PlanID.MinRole()) plan.buildAuthorized() if plan.PlanID.IsSelect() { if plan.FieldQuery != nil { @@ -406,7 +404,6 @@ func (qe *QueryEngine) GetStreamPlan(sql string) (*TabletPlan, error) { } plan := &TabletPlan{Plan: splan} plan.Rules = qe.queryRuleSources.FilterByPlan(sql, plan.PlanID, plan.TableName().String()) - plan.LegacyAuthorized = tableacl.Authorized(plan.TableName().String(), plan.PlanID.MinRole()) plan.buildAuthorized() return plan, nil } @@ -421,7 +418,6 @@ func (qe *QueryEngine) GetMessageStreamPlan(name string) (*TabletPlan, error) { } plan := &TabletPlan{Plan: splan} plan.Rules = qe.queryRuleSources.FilterByPlan("stream from "+name, plan.PlanID, plan.TableName().String()) - plan.LegacyAuthorized = tableacl.Authorized(plan.TableName().String(), plan.PlanID.MinRole()) plan.buildAuthorized() return plan, nil } diff --git a/go/vt/vttablet/tabletserver/query_executor.go b/go/vt/vttablet/tabletserver/query_executor.go index 7576f920a85..69d0079d90f 100644 --- a/go/vt/vttablet/tabletserver/query_executor.go +++ b/go/vt/vttablet/tabletserver/query_executor.go @@ -17,7 +17,6 @@ limitations under the License. package tabletserver import ( - "flag" "fmt" "io" "strings" @@ -45,9 +44,6 @@ import ( vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) -// TODO(sougou): remove after affected parties have transitioned to new behavior. -var legacyTableACL = flag.Bool("legacy-table-acl", false, "deprecated: this flag can be used to revert to the older table ACL behavior, which checked access for at most one table") - // QueryExecutor is used for executing a query request. type QueryExecutor struct { query string @@ -373,15 +369,9 @@ func (qre *QueryExecutor) checkPermissions() error { return nil } - if *legacyTableACL { - if !qre.plan.TableName().IsEmpty() { - return qre.checkAccess(qre.plan.LegacyAuthorized, qre.plan.TableName().String(), callerID) - } - } else { - for i, auth := range qre.plan.Authorized { - if err := qre.checkAccess(auth, qre.plan.Permissions[i].TableName, callerID); err != nil { - return err - } + for i, auth := range qre.plan.Authorized { + if err := qre.checkAccess(auth, qre.plan.Permissions[i].TableName, callerID); err != nil { + return err } } diff --git a/go/vt/vttest/local_cluster.go b/go/vt/vttest/local_cluster.go index b69c2ccec74..0ae9495a8cb 100644 --- a/go/vt/vttest/local_cluster.go +++ b/go/vt/vttest/local_cluster.go @@ -99,6 +99,8 @@ type Config struct { // TransactionMode is SINGLE, MULTI or TWOPC TransactionMode string + + TransactionTimeout float64 } // InitSchemas is a shortcut for tests that just want to setup a single diff --git a/go/vt/vttest/vtprocess.go b/go/vt/vttest/vtprocess.go index e3065477033..91bf375a595 100644 --- a/go/vt/vttest/vtprocess.go +++ b/go/vt/vttest/vtprocess.go @@ -237,6 +237,9 @@ func VtcomboProcess(env Environment, args *Config, mysql MySQLManager) *VtProces if args.TransactionMode != "" { vt.ExtraArgs = append(vt.ExtraArgs, []string{"-transaction_mode", args.TransactionMode}...) } + if args.TransactionTimeout != 0 { + vt.ExtraArgs = append(vt.ExtraArgs, "-queryserver-config-transaction-timeout", fmt.Sprintf("%f", args.TransactionTimeout)) + } if socket != "" { vt.ExtraArgs = append(vt.ExtraArgs, []string{ diff --git a/go/vt/worker/vertical_split_clone_cmd.go b/go/vt/worker/vertical_split_clone_cmd.go index f221ee5d677..4f201036458 100644 --- a/go/vt/worker/vertical_split_clone_cmd.go +++ b/go/vt/worker/vertical_split_clone_cmd.go @@ -82,13 +82,20 @@ const verticalSplitCloneHTML2 = `

- -
+ + + +


+ + ?
@@ -107,7 +114,8 @@ func commandVerticalSplitClone(wi *Instance, wr *wrangler.Wrangler, subFlags *fl writeQueryMaxRows := subFlags.Int("write_query_max_rows", defaultWriteQueryMaxRows, "maximum number of rows per write query") writeQueryMaxSize := subFlags.Int("write_query_max_size", defaultWriteQueryMaxSize, "maximum size (in bytes) per write query") destinationWriterCount := subFlags.Int("destination_writer_count", defaultDestinationWriterCount, "number of concurrent RPCs to execute on the destination") - minHealthyRdonlyTablets := subFlags.Int("min_healthy_rdonly_tablets", defaultMinHealthyTablets, "minimum number of healthy RDONLY tablets before taking out one") + minHealthyTablets := subFlags.Int("min_healthy_tablets", defaultMinHealthyTablets, "minimum number of healthy tablets before taking out one") + useConsistentSnapshot := subFlags.Bool("use_consistent_snapshot", defaultUseConsistentSnapshot, "Instead of pausing replication on the source, uses transactions with consistent snapshot to have a stable view of the data.") tabletTypeStr := subFlags.String("tablet_type", "RDONLY", "tablet type to use (RDONLY or REPLICA)") maxTPS := subFlags.Int64("max_tps", defaultMaxTPS, "if non-zero, limit copy to maximum number of (write) transactions/second on the destination (unlimited by default)") maxReplicationLag := subFlags.Int64("max_replication_lag", defaultMaxReplicationLag, "if set, the adapative throttler will be enabled and automatically adjust the write rate to keep the lag below the set value in seconds (disabled by default)") @@ -134,7 +142,7 @@ func commandVerticalSplitClone(wi *Instance, wr *wrangler.Wrangler, subFlags *fl return nil, fmt.Errorf("command SplitClone invalid tablet_type: %v", tabletType) } - worker, err := newVerticalSplitCloneWorker(wr, wi.cell, keyspace, shard, *online, *offline, tableArray, *chunkCount, *minRowsPerChunk, *sourceReaderCount, *writeQueryMaxRows, *writeQueryMaxSize, *destinationWriterCount, *minHealthyRdonlyTablets, topodata.TabletType(tabletType), *maxTPS, *maxReplicationLag, /*useConsistentSnapshot*/false) + worker, err := newVerticalSplitCloneWorker(wr, wi.cell, keyspace, shard, *online, *offline, tableArray, *chunkCount, *minRowsPerChunk, *sourceReaderCount, *writeQueryMaxRows, *writeQueryMaxSize, *destinationWriterCount, *minHealthyTablets, topodata.TabletType(tabletType), *maxTPS, *maxReplicationLag, *useConsistentSnapshot) if err != nil { return nil, vterrors.Wrap(err, "cannot create worker") } @@ -215,9 +223,10 @@ func interactiveVerticalSplitClone(ctx context.Context, wi *Instance, wr *wrangl result["DefaultWriteQueryMaxRows"] = fmt.Sprintf("%v", defaultWriteQueryMaxRows) result["DefaultWriteQueryMaxSize"] = fmt.Sprintf("%v", defaultWriteQueryMaxSize) result["DefaultDestinationWriterCount"] = fmt.Sprintf("%v", defaultDestinationWriterCount) - result["DefaultMinHealthyRdonlyTablets"] = fmt.Sprintf("%v", defaultMinHealthyTablets) + result["DefaultMinHealthyTablets"] = fmt.Sprintf("%v", defaultMinHealthyTablets) result["DefaultMaxTPS"] = fmt.Sprintf("%v", defaultMaxTPS) result["DefaultMaxReplicationLag"] = fmt.Sprintf("%v", defaultMaxReplicationLag) + result["DefaultUseConsistentSnapshot"] = fmt.Sprintf("%v", defaultUseConsistentSnapshot) return nil, verticalSplitCloneTemplate2, result, nil } tableArray := strings.Split(tables, ",") @@ -257,10 +266,10 @@ func interactiveVerticalSplitClone(ctx context.Context, wi *Instance, wr *wrangl if err != nil { return nil, nil, nil, vterrors.Wrap(err, "cannot parse destinationWriterCount") } - minHealthyRdonlyTabletsStr := r.FormValue("minHealthyRdonlyTablets") - minHealthyRdonlyTablets, err := strconv.ParseInt(minHealthyRdonlyTabletsStr, 0, 64) + minHealthyTabletsStr := r.FormValue("minHealthyTablets") + minHealthyTablets, err := strconv.ParseInt(minHealthyTabletsStr, 0, 64) if err != nil { - return nil, nil, nil, vterrors.Wrap(err, "cannot parse minHealthyRdonlyTablets") + return nil, nil, nil, vterrors.Wrap(err, "cannot parse minHealthyTablets") } maxTPSStr := r.FormValue("maxTPS") maxTPS, err := strconv.ParseInt(maxTPSStr, 0, 64) @@ -278,7 +287,6 @@ func interactiveVerticalSplitClone(ctx context.Context, wi *Instance, wr *wrangl return nil, nil, nil, fmt.Errorf("cannot parse tabletType: %v", tabletType) } - // Figure out the shard shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) shardMap, err := wr.TopoServer().FindAllShardsInKeyspace(shortCtx, keyspace) @@ -294,11 +302,14 @@ func interactiveVerticalSplitClone(ctx context.Context, wi *Instance, wr *wrangl shard = s } + useConsistentSnapshotStr := r.FormValue("useConsistentSnapshot") + useConsistentSnapshot := useConsistentSnapshotStr == "true" + // start the clone job wrk, err := newVerticalSplitCloneWorker(wr, wi.cell, keyspace, shard, online, offline, tableArray, int(chunkCount), int(minRowsPerChunk), int(sourceReaderCount), int(writeQueryMaxRows), int(writeQueryMaxSize), - int(destinationWriterCount), int(minHealthyRdonlyTablets), topodata.TabletType(tabletType), maxTPS, - maxReplicationLag, false) + int(destinationWriterCount), int(minHealthyTablets), topodata.TabletType(tabletType), maxTPS, + maxReplicationLag, useConsistentSnapshot) if err != nil { return nil, nil, nil, vterrors.Wrap(err, "cannot create worker") } diff --git a/go/vt/worker/vertical_split_clone_test.go b/go/vt/worker/vertical_split_clone_test.go index ad351c3363b..3656f405b1d 100644 --- a/go/vt/worker/vertical_split_clone_test.go +++ b/go/vt/worker/vertical_split_clone_test.go @@ -181,7 +181,7 @@ func TestVerticalSplitClone(t *testing.T) { "-min_rows_per_chunk", "10", "-destination_writer_count", "10", // This test uses only one healthy RDONLY tablet. - "-min_healthy_rdonly_tablets", "1", + "-min_healthy_tablets", "1", "destination_ks/0", } if err := runCommand(t, wi, wi.wr, args); err != nil { diff --git a/go/vt/wrangler/keyspace.go b/go/vt/wrangler/keyspace.go index 759a3d47985..ac511f1ed0b 100644 --- a/go/vt/wrangler/keyspace.go +++ b/go/vt/wrangler/keyspace.go @@ -146,7 +146,7 @@ func (wr *Wrangler) printShards(ctx context.Context, si []*topo.ShardInfo) error wr.Logger().Printf(" %v\n", row) } } - wr.Logger().Printf(" Served Types: %v\n", si.IsMasterServing) + wr.Logger().Printf(" Is Master Serving: %v\n", si.IsMasterServing) if len(si.TabletControls) != 0 { wr.Logger().Printf(" Tablet Controls: %v\n", si.TabletControls) } diff --git a/go/vt/wrangler/testlib/migrate_served_types_test.go b/go/vt/wrangler/testlib/migrate_served_types_test.go index 4d118afd7f8..5fb5ecf4a11 100644 --- a/go/vt/wrangler/testlib/migrate_served_types_test.go +++ b/go/vt/wrangler/testlib/migrate_served_types_test.go @@ -313,18 +313,6 @@ func TestMultiShardMigrateServedTypes(t *testing.T) { dest4Rdonly := NewFakeTablet(t, wr, "cell1", 72, topodatapb.TabletType_RDONLY, nil, TabletKeyspaceShard(t, "ks", "c0-")) - if source1Master == nil || source1Replica == nil || source1Rdonly == nil || source2Master == nil || source2Replica == nil || source2Rdonly == nil { - - } - - if dest1Master == nil || dest1Replica == nil || dest1Rdonly == nil || dest2Master == nil || dest2Replica == nil || dest2Rdonly == nil { - - } - - if dest3Master == nil || dest3Replica == nil || dest3Rdonly == nil || dest4Master == nil || dest4Replica == nil || dest4Rdonly == nil { - - } - // Build keyspace graph err := topotools.RebuildKeyspace(context.Background(), logutil.NewConsoleLogger(), ts, "ks", []string{"cell1"}) if err != nil { diff --git a/go/vt/wrangler/testlib/permissions_test.go b/go/vt/wrangler/testlib/permissions_test.go index 972ca4fe05a..70a8a94b7e5 100644 --- a/go/vt/wrangler/testlib/permissions_test.go +++ b/go/vt/wrangler/testlib/permissions_test.go @@ -55,7 +55,7 @@ func TestPermissions(t *testing.T) { // master will be asked for permissions master.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{ - "SELECT * FROM mysql.user": { + "SELECT * FROM mysql.user ORDER BY host, user": { Fields: []*querypb.Field{ { Name: "Host", @@ -419,7 +419,7 @@ func TestPermissions(t *testing.T) { }, }, }, - "SELECT * FROM mysql.db": { + "SELECT * FROM mysql.db ORDER BY host, db, user": { Fields: []*querypb.Field{ { Name: "Host", @@ -544,19 +544,19 @@ func TestPermissions(t *testing.T) { defer master.StopActionLoop(t) // Make a two-level-deep copy, so we can make them diverge later. - user := *master.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"] + user := *master.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user ORDER BY host, user"] user.Fields = append([]*querypb.Field{}, user.Fields...) // replica will be asked for permissions replica.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{ - "SELECT * FROM mysql.user": &user, - "SELECT * FROM mysql.db": master.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.db"], + "SELECT * FROM mysql.user ORDER BY host, user": &user, + "SELECT * FROM mysql.db ORDER BY host, db, user": master.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.db ORDER BY host, db, user"], } replica.StartActionLoop(t, wr) defer replica.StopActionLoop(t) // Overwrite with the correct value to make sure it passes. - replica.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"].Fields[0] = &querypb.Field{ + replica.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user ORDER BY host, user"].Fields[0] = &querypb.Field{ Name: "Host", Type: sqltypes.Char, } @@ -567,7 +567,7 @@ func TestPermissions(t *testing.T) { } // modify one field, this should fail - replica.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user"].Fields[0] = &querypb.Field{ + replica.FakeMysqlDaemon.FetchSuperQueryMap["SELECT * FROM mysql.user ORDER BY host, user"].Fields[0] = &querypb.Field{ Name: "Wrong", Type: sqltypes.Char, } diff --git a/go/vt/zkctl/zksrv.sh b/go/vt/zkctl/zksrv.sh index 30f4f2e1ef4..62172c5d279 100755 --- a/go/vt/zkctl/zksrv.sh +++ b/go/vt/zkctl/zksrv.sh @@ -1,13 +1,13 @@ #!/bin/bash # Copyright 2017 Google Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,7 +21,7 @@ logdir="$1" config="$2" pidfile="$3" -zk_ver=3.4.13 +zk_ver=${ZK_VERSION:-3.4.14} classpath="$VTROOT/dist/vt-zookeeper-$zk_ver/lib/zookeeper-$zk_ver-fatjar.jar:/usr/local/lib/zookeeper-$zk_ver-fatjar.jar:/usr/share/java/zookeeper-$zk_ver.jar" mkdir -p "$logdir" @@ -60,7 +60,7 @@ $cmd < /dev/null &> /dev/null & pid=$! log "INFO pid: $pid pidfile: $pidfile" -if [ "$pidfile" ]; then +if [ "$pidfile" ]; then if [ -f "$pidfile" ]; then rm "$pidfile" fi diff --git a/helm/vitess/templates/_etcd.tpl b/helm/vitess/templates/_etcd.tpl index b30ce7c09c9..ad701cc6123 100644 --- a/helm/vitess/templates/_etcd.tpl +++ b/helm/vitess/templates/_etcd.tpl @@ -7,6 +7,7 @@ {{- $replicas := index . 1 -}} {{- $version := index . 2 -}} {{- $resources := index . 3 }} +{{- $clusterWide := index . 4 }} ################################### # EtcdCluster @@ -15,6 +16,12 @@ apiVersion: "etcd.database.coreos.com/v1beta2" kind: "EtcdCluster" metadata: name: "etcd-{{ $name }}" + ## Adding this annotation make this cluster managed by clusterwide operators + ## namespaced operators ignore it + annotations: + {{ if $clusterWide }} + etcd.database.coreos.com/scope: clusterwide + {{ end }} spec: size: {{ $replicas }} version: {{ $version | quote }} diff --git a/helm/vitess/templates/vitess.yaml b/helm/vitess/templates/vitess.yaml index 67afbd43fc7..7c3d36aac25 100644 --- a/helm/vitess/templates/vitess.yaml +++ b/helm/vitess/templates/vitess.yaml @@ -29,8 +29,9 @@ {{- $replicas := $.Values.topology.globalCell.replicas | default $.Values.etcd.replicas -}} {{- $version := $.Values.topology.globalCell.version | default $.Values.etcd.version -}} {{- $resources := $.Values.topology.globalCell.resources | default $.Values.etcd.resources -}} +{{- $clusterWide := $.Values.topology.globalCell.resources | default $.Values.etcd.clusterWide -}} -{{ include "etcd" (tuple "global" $replicas $version $resources) }} +{{ include "etcd" (tuple "global" $replicas $version $resources $clusterWide) }} # Create requested resources in each cell. {{ range $cell := $.Values.topology.cells }} @@ -41,8 +42,9 @@ {{- $replicas := $cell.etcd.replicas | default $.Values.etcd.replicas -}} {{- $version := $cell.etcd.version | default $.Values.etcd.version -}} {{- $resources := $cell.etcd.resources | default $.Values.etcd.resources -}} +{{- $clusterWide := $cell.etcd.clusterWide | default $.Values.etcd.clusterWide -}} -{{ include "etcd" (tuple $cellClean $replicas $version $resources) }} +{{ include "etcd" (tuple $cellClean $replicas $version $resources $clusterWide) }} --- # create one controller per cell {{ include "vtctld" (tuple $.Values.topology $cell $.Values.vtctld $.Release.Namespace $.Values.config) }} diff --git a/helm/vitess/values.yaml b/helm/vitess/values.yaml index c4e5563a464..9ad16ad0825 100644 --- a/helm/vitess/values.yaml +++ b/helm/vitess/values.yaml @@ -173,6 +173,9 @@ etcd: requests: cpu: 200m memory: 100Mi + # If clusterWide is set to true, will add an annotation to EtcdCluster + # to make this cluster managed by clusterwide operators + # clusterWide: true # Default values for vtctld resources defined in 'topology' vtctld: diff --git a/proto/topodata.proto b/proto/topodata.proto index 870c794895a..2677f721911 100644 --- a/proto/topodata.proto +++ b/proto/topodata.proto @@ -347,7 +347,12 @@ message CellInfo { // to server_address. string root = 2; - // Region is a group this cell belongs to. Used by vtgate to route traffic to - // other cells (in same region) when there is no available tablet in the current cell. - string region = 3; + // OBSOLETE: region 3 + reserved 3; +} + +// CellsAlias +message CellsAlias { + // Cells that map to this alias + repeated string cells = 2; } diff --git a/py/vtproto/topodata_pb2.py b/py/vtproto/topodata_pb2.py index 7970c86e338..66ac9fea9b0 100644 --- a/py/vtproto/topodata_pb2.py +++ b/py/vtproto/topodata_pb2.py @@ -20,7 +20,7 @@ package='topodata', syntax='proto3', serialized_options=_b('\n\017io.vitess.protoZ%vitess.io/vitess/go/vt/proto/topodata'), - serialized_pb=_b('\n\x0etopodata.proto\x12\x08topodata\"&\n\x08KeyRange\x12\r\n\x05start\x18\x01 \x01(\x0c\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x0c\"(\n\x0bTabletAlias\x12\x0c\n\x04\x63\x65ll\x18\x01 \x01(\t\x12\x0b\n\x03uid\x18\x02 \x01(\r\"\xb6\x03\n\x06Tablet\x12$\n\x05\x61lias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12\x10\n\x08hostname\x18\x02 \x01(\t\x12/\n\x08port_map\x18\x04 \x03(\x0b\x32\x1d.topodata.Tablet.PortMapEntry\x12\x10\n\x08keyspace\x18\x05 \x01(\t\x12\r\n\x05shard\x18\x06 \x01(\t\x12%\n\tkey_range\x18\x07 \x01(\x0b\x32\x12.topodata.KeyRange\x12\"\n\x04type\x18\x08 \x01(\x0e\x32\x14.topodata.TabletType\x12\x18\n\x10\x64\x62_name_override\x18\t \x01(\t\x12(\n\x04tags\x18\n \x03(\x0b\x32\x1a.topodata.Tablet.TagsEntry\x12\x16\n\x0emysql_hostname\x18\x0c \x01(\t\x12\x12\n\nmysql_port\x18\r \x01(\x05\x1a.\n\x0cPortMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04J\x04\x08\x0b\x10\x0c\"\xd3\x04\n\x05Shard\x12+\n\x0cmaster_alias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x30\n\x0cserved_types\x18\x03 \x03(\x0b\x32\x1a.topodata.Shard.ServedType\x12\x32\n\rsource_shards\x18\x04 \x03(\x0b\x32\x1b.topodata.Shard.SourceShard\x12\x36\n\x0ftablet_controls\x18\x06 \x03(\x0b\x32\x1d.topodata.Shard.TabletControl\x12\x19\n\x11is_master_serving\x18\x07 \x01(\x08\x1a\x46\n\nServedType\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x1ar\n\x0bSourceShard\x12\x0b\n\x03uid\x18\x01 \x01(\r\x12\x10\n\x08keyspace\x18\x02 \x01(\t\x12\r\n\x05shard\x18\x03 \x01(\t\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x1a{\n\rTabletControl\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x12\x1a\n\x12\x62lacklisted_tables\x18\x04 \x03(\t\x12\x0e\n\x06\x66rozen\x18\x05 \x01(\x08J\x04\x08\x03\x10\x04J\x04\x08\x05\x10\x06\"\xf5\x01\n\x08Keyspace\x12\x1c\n\x14sharding_column_name\x18\x01 \x01(\t\x12\x36\n\x14sharding_column_type\x18\x02 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12\x33\n\x0cserved_froms\x18\x04 \x03(\x0b\x32\x1d.topodata.Keyspace.ServedFrom\x1aX\n\nServedFrom\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x12\x10\n\x08keyspace\x18\x03 \x01(\tJ\x04\x08\x03\x10\x04\"w\n\x10ShardReplication\x12.\n\x05nodes\x18\x01 \x03(\x0b\x32\x1f.topodata.ShardReplication.Node\x1a\x33\n\x04Node\x12+\n\x0ctablet_alias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\"E\n\x0eShardReference\x12\x0c\n\x04name\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\"i\n\x12ShardTabletControl\x12\x0c\n\x04name\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x1e\n\x16query_service_disabled\x18\x03 \x01(\x08\"\xda\x03\n\x0bSrvKeyspace\x12;\n\npartitions\x18\x01 \x03(\x0b\x32\'.topodata.SrvKeyspace.KeyspacePartition\x12\x1c\n\x14sharding_column_name\x18\x02 \x01(\t\x12\x36\n\x14sharding_column_type\x18\x03 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12\x35\n\x0bserved_from\x18\x04 \x03(\x0b\x32 .topodata.SrvKeyspace.ServedFrom\x1a\xaf\x01\n\x11KeyspacePartition\x12)\n\x0bserved_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\x32\n\x10shard_references\x18\x02 \x03(\x0b\x32\x18.topodata.ShardReference\x12;\n\x15shard_tablet_controls\x18\x03 \x03(\x0b\x32\x1c.topodata.ShardTabletControl\x1aI\n\nServedFrom\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\x10\n\x08keyspace\x18\x02 \x01(\tJ\x04\x08\x05\x10\x06\"@\n\x08\x43\x65llInfo\x12\x16\n\x0eserver_address\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x02 \x01(\t\x12\x0e\n\x06region\x18\x03 \x01(\t*2\n\x0eKeyspaceIdType\x12\t\n\x05UNSET\x10\x00\x12\n\n\x06UINT64\x10\x01\x12\t\n\x05\x42YTES\x10\x02*\x90\x01\n\nTabletType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06MASTER\x10\x01\x12\x0b\n\x07REPLICA\x10\x02\x12\n\n\x06RDONLY\x10\x03\x12\t\n\x05\x42\x41TCH\x10\x03\x12\t\n\x05SPARE\x10\x04\x12\x10\n\x0c\x45XPERIMENTAL\x10\x05\x12\n\n\x06\x42\x41\x43KUP\x10\x06\x12\x0b\n\x07RESTORE\x10\x07\x12\x0b\n\x07\x44RAINED\x10\x08\x1a\x02\x10\x01\x42\x38\n\x0fio.vitess.protoZ%vitess.io/vitess/go/vt/proto/topodatab\x06proto3') + serialized_pb=_b('\n\x0etopodata.proto\x12\x08topodata\"&\n\x08KeyRange\x12\r\n\x05start\x18\x01 \x01(\x0c\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x0c\"(\n\x0bTabletAlias\x12\x0c\n\x04\x63\x65ll\x18\x01 \x01(\t\x12\x0b\n\x03uid\x18\x02 \x01(\r\"\xb6\x03\n\x06Tablet\x12$\n\x05\x61lias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12\x10\n\x08hostname\x18\x02 \x01(\t\x12/\n\x08port_map\x18\x04 \x03(\x0b\x32\x1d.topodata.Tablet.PortMapEntry\x12\x10\n\x08keyspace\x18\x05 \x01(\t\x12\r\n\x05shard\x18\x06 \x01(\t\x12%\n\tkey_range\x18\x07 \x01(\x0b\x32\x12.topodata.KeyRange\x12\"\n\x04type\x18\x08 \x01(\x0e\x32\x14.topodata.TabletType\x12\x18\n\x10\x64\x62_name_override\x18\t \x01(\t\x12(\n\x04tags\x18\n \x03(\x0b\x32\x1a.topodata.Tablet.TagsEntry\x12\x16\n\x0emysql_hostname\x18\x0c \x01(\t\x12\x12\n\nmysql_port\x18\r \x01(\x05\x1a.\n\x0cPortMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04J\x04\x08\x0b\x10\x0c\"\xd3\x04\n\x05Shard\x12+\n\x0cmaster_alias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x30\n\x0cserved_types\x18\x03 \x03(\x0b\x32\x1a.topodata.Shard.ServedType\x12\x32\n\rsource_shards\x18\x04 \x03(\x0b\x32\x1b.topodata.Shard.SourceShard\x12\x36\n\x0ftablet_controls\x18\x06 \x03(\x0b\x32\x1d.topodata.Shard.TabletControl\x12\x19\n\x11is_master_serving\x18\x07 \x01(\x08\x1a\x46\n\nServedType\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x1ar\n\x0bSourceShard\x12\x0b\n\x03uid\x18\x01 \x01(\r\x12\x10\n\x08keyspace\x18\x02 \x01(\t\x12\r\n\x05shard\x18\x03 \x01(\t\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x1a{\n\rTabletControl\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x12\x1a\n\x12\x62lacklisted_tables\x18\x04 \x03(\t\x12\x0e\n\x06\x66rozen\x18\x05 \x01(\x08J\x04\x08\x03\x10\x04J\x04\x08\x05\x10\x06\"\xf5\x01\n\x08Keyspace\x12\x1c\n\x14sharding_column_name\x18\x01 \x01(\t\x12\x36\n\x14sharding_column_type\x18\x02 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12\x33\n\x0cserved_froms\x18\x04 \x03(\x0b\x32\x1d.topodata.Keyspace.ServedFrom\x1aX\n\nServedFrom\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x12\x10\n\x08keyspace\x18\x03 \x01(\tJ\x04\x08\x03\x10\x04\"w\n\x10ShardReplication\x12.\n\x05nodes\x18\x01 \x03(\x0b\x32\x1f.topodata.ShardReplication.Node\x1a\x33\n\x04Node\x12+\n\x0ctablet_alias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\"E\n\x0eShardReference\x12\x0c\n\x04name\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\"i\n\x12ShardTabletControl\x12\x0c\n\x04name\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x1e\n\x16query_service_disabled\x18\x03 \x01(\x08\"\xda\x03\n\x0bSrvKeyspace\x12;\n\npartitions\x18\x01 \x03(\x0b\x32\'.topodata.SrvKeyspace.KeyspacePartition\x12\x1c\n\x14sharding_column_name\x18\x02 \x01(\t\x12\x36\n\x14sharding_column_type\x18\x03 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12\x35\n\x0bserved_from\x18\x04 \x03(\x0b\x32 .topodata.SrvKeyspace.ServedFrom\x1a\xaf\x01\n\x11KeyspacePartition\x12)\n\x0bserved_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\x32\n\x10shard_references\x18\x02 \x03(\x0b\x32\x18.topodata.ShardReference\x12;\n\x15shard_tablet_controls\x18\x03 \x03(\x0b\x32\x1c.topodata.ShardTabletControl\x1aI\n\nServedFrom\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\x10\n\x08keyspace\x18\x02 \x01(\tJ\x04\x08\x05\x10\x06\"6\n\x08\x43\x65llInfo\x12\x16\n\x0eserver_address\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x02 \x01(\tJ\x04\x08\x03\x10\x04\"\x1b\n\nCellsAlias\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t*2\n\x0eKeyspaceIdType\x12\t\n\x05UNSET\x10\x00\x12\n\n\x06UINT64\x10\x01\x12\t\n\x05\x42YTES\x10\x02*\x90\x01\n\nTabletType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06MASTER\x10\x01\x12\x0b\n\x07REPLICA\x10\x02\x12\n\n\x06RDONLY\x10\x03\x12\t\n\x05\x42\x41TCH\x10\x03\x12\t\n\x05SPARE\x10\x04\x12\x10\n\x0c\x45XPERIMENTAL\x10\x05\x12\n\n\x06\x42\x41\x43KUP\x10\x06\x12\x0b\n\x07RESTORE\x10\x07\x12\x0b\n\x07\x44RAINED\x10\x08\x1a\x02\x10\x01\x42\x38\n\x0fio.vitess.protoZ%vitess.io/vitess/go/vt/proto/topodatab\x06proto3') ) _KEYSPACEIDTYPE = _descriptor.EnumDescriptor( @@ -44,8 +44,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=2239, - serialized_end=2289, + serialized_start=2258, + serialized_end=2308, ) _sym_db.RegisterEnumDescriptor(_KEYSPACEIDTYPE) @@ -99,8 +99,8 @@ ], containing_type=None, serialized_options=_b('\020\001'), - serialized_start=2292, - serialized_end=2436, + serialized_start=2311, + serialized_end=2455, ) _sym_db.RegisterEnumDescriptor(_TABLETTYPE) @@ -971,10 +971,34 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2173, + serialized_end=2227, +) + + +_CELLSALIAS = _descriptor.Descriptor( + name='CellsAlias', + full_name='topodata.CellsAlias', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ _descriptor.FieldDescriptor( - name='region', full_name='topodata.CellInfo.region', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + name='cells', full_name='topodata.CellsAlias.cells', index=0, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -990,8 +1014,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2173, - serialized_end=2237, + serialized_start=2229, + serialized_end=2256, ) _TABLET_PORTMAPENTRY.containing_type = _TABLET @@ -1040,6 +1064,7 @@ DESCRIPTOR.message_types_by_name['ShardTabletControl'] = _SHARDTABLETCONTROL DESCRIPTOR.message_types_by_name['SrvKeyspace'] = _SRVKEYSPACE DESCRIPTOR.message_types_by_name['CellInfo'] = _CELLINFO +DESCRIPTOR.message_types_by_name['CellsAlias'] = _CELLSALIAS DESCRIPTOR.enum_types_by_name['KeyspaceIdType'] = _KEYSPACEIDTYPE DESCRIPTOR.enum_types_by_name['TabletType'] = _TABLETTYPE _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -1186,6 +1211,13 @@ )) _sym_db.RegisterMessage(CellInfo) +CellsAlias = _reflection.GeneratedProtocolMessageType('CellsAlias', (_message.Message,), dict( + DESCRIPTOR = _CELLSALIAS, + __module__ = 'topodata_pb2' + # @@protoc_insertion_point(class_scope:topodata.CellsAlias) + )) +_sym_db.RegisterMessage(CellsAlias) + DESCRIPTOR._options = None _TABLETTYPE._options = None diff --git a/test/cell_aliases.py b/test/cell_aliases.py new file mode 100755 index 00000000000..bb910daadb4 --- /dev/null +++ b/test/cell_aliases.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This test cell aliases feature + +We start with no aliases and assert that vtgates can't route to replicas/rondly tablets. +Then we add an alias, and these tablets should be routable + +""" + +import threading +import time + +import logging +import unittest + +import base_sharding +import environment +import tablet +import utils + +from vtproto import topodata_pb2 +from vtdb import keyrange_constants +from vtdb import vtgate_client + +use_alias = False + +# initial shards +# range '' - 80 +shard_0_master = tablet.Tablet() +shard_0_replica = tablet.Tablet(cell='ny') +shard_0_rdonly = tablet.Tablet(cell='ny') + +#shard_0_replica = tablet.Tablet() +#shard_0_rdonly = tablet.Tablet() +# range 80 - '' +shard_1_master = tablet.Tablet() +#shard_1_replica = tablet.Tablet() +#shard_1_rdonly = tablet.Tablet() + +shard_1_replica = tablet.Tablet(cell='ny') +shard_1_rdonly = tablet.Tablet(cell='ny') + +all_tablets = ([shard_0_master, shard_0_replica, shard_0_rdonly, + shard_1_master, shard_1_replica,shard_1_rdonly]) + +vschema = { + 'test_keyspace': '''{ + "sharded": true, + "vindexes": { + "hash_index": { + "type": "hash" + } + }, + "tables": { + "test_table": { + "column_vindexes": [ + { + "column": "custom_ksid_col", + "name": "hash_index" + } + ] + } + } + }''', +} + +def setUpModule(): + try: + environment.topo_server().setup() + setup_procs = [t.init_mysql() + for t in all_tablets] + utils.Vtctld().start() + utils.wait_procs(setup_procs) + except: + tearDownModule() + raise + +def tearDownModule(): + utils.required_teardown() + if utils.options.skip_teardown: + return + + teardown_procs = [t.teardown_mysql() for t in all_tablets] + utils.wait_procs(teardown_procs, raise_on_error=False) + environment.topo_server().teardown() + utils.kill_sub_processes() + utils.remove_tmp_files() + for t in all_tablets: + t.remove_tree() + +class TestCellsAliases(unittest.TestCase, base_sharding.BaseShardingTest): + + int_type = 265 + + # Gets a vtgate connection + def _get_connection(self, timeout=10.0): + protocol, endpoint = utils.vtgate.rpc_endpoint(python=True) + try: + return vtgate_client.connect(protocol, endpoint, timeout) + except Exception: + logging.exception('Connection to vtgate (timeout=%s) failed.', timeout) + raise + + # executes requetest in tablet type + def _execute_on_tablet_type(self, vtgate_conn, tablet_type, sql, bind_vars): + return vtgate_conn._execute( + sql, bind_vars, tablet_type=tablet_type, keyspace_name=None) + + + # create_schema will create the same schema on the keyspace + # then insert some values + def _create_schema(self): + if base_sharding.keyspace_id_type == keyrange_constants.KIT_BYTES: + t = 'varbinary(64)' + else: + t = 'bigint(20) unsigned' + # Note that the primary key columns are not defined first on purpose to test + # that a reordered column list is correctly used everywhere in vtworker. + create_table_template = '''create table %s( +custom_ksid_col ''' + t + ''' not null, +msg varchar(64), +id bigint not null, +parent_id bigint not null, +primary key (parent_id, id), +index by_msg (msg) +) Engine=InnoDB''' + + utils.run_vtctl(['ApplySchema', + '-sql=' + create_table_template % ('test_table'), + 'test_keyspace'], + auto_log=True) + + def _insert_startup_values(self): + self._insert_value(shard_0_master, 'test_table', 1, 'msg1', + 0x1000000000000000) + self._insert_value(shard_1_master, 'test_table', 2, 'msg2', + 0x9000000000000000) + self._insert_value(shard_1_master, 'test_table', 3, 'msg3', + 0xD000000000000000) + + def test_cells_aliases(self): + utils.run_vtctl(['CreateKeyspace', + '--sharding_column_name', 'custom_ksid_col', + '--sharding_column_type', base_sharding.keyspace_id_type, + 'test_keyspace']) + + shard_0_master.init_tablet('replica', 'test_keyspace', '-80') + shard_0_replica.init_tablet('replica', 'test_keyspace', '-80') + shard_0_rdonly.init_tablet('rdonly', 'test_keyspace', '-80') + shard_1_master.init_tablet('replica', 'test_keyspace', '80-') + shard_1_replica.init_tablet('replica', 'test_keyspace', '80-') + shard_1_rdonly.init_tablet('rdonly', 'test_keyspace', '80-') + + utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) + ks = utils.run_vtctl_json(['GetSrvKeyspace', 'test_nj', 'test_keyspace']) + self.assertEqual(ks['sharding_column_name'], 'custom_ksid_col') + + # we set full_mycnf_args to True as a test in the KIT_BYTES case + full_mycnf_args = (base_sharding.keyspace_id_type == + keyrange_constants.KIT_BYTES) + + # create databases so vttablet can start behaving somewhat normally + for t in [shard_0_master, shard_0_replica, shard_0_rdonly, + shard_1_master, shard_1_replica, shard_1_rdonly]: + t.create_db('vt_test_keyspace') + t.start_vttablet(wait_for_state=None, full_mycnf_args=full_mycnf_args, + binlog_use_v3_resharding_mode=False) + + # wait for the tablets (replication is not setup, they won't be healthy) + for t in [shard_0_master, shard_0_replica, shard_0_rdonly, + shard_1_master, shard_1_replica, shard_1_rdonly]: + t.wait_for_vttablet_state('NOT_SERVING') + + # reparent to make the tablets work + utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/-80', + shard_0_master.tablet_alias], auto_log=True) + utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/80-', + shard_1_master.tablet_alias], auto_log=True) + + # check the shards + shards = utils.run_vtctl_json(['FindAllShardsInKeyspace', 'test_keyspace']) + self.assertIn('-80', shards, 'unexpected shards: %s' % str(shards)) + self.assertIn('80-', shards, 'unexpected shards: %s' % str(shards)) + self.assertEqual(len(shards), 2, 'unexpected shards: %s' % str(shards)) + + # create the tables + self._create_schema() + self._insert_startup_values() + + + utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) + + # Make sure srv keyspace graph looks as expected + utils.check_srv_keyspace( + 'test_nj', 'test_keyspace', + 'Partitions(master): -80 80-\n' + 'Partitions(rdonly): -80 80-\n' + 'Partitions(replica): -80 80-\n', + keyspace_id_type=base_sharding.keyspace_id_type, + sharding_column_name='custom_ksid_col') + + utils.check_srv_keyspace( + 'test_ny', 'test_keyspace', + 'Partitions(master): -80 80-\n' + 'Partitions(rdonly): -80 80-\n' + 'Partitions(replica): -80 80-\n', + keyspace_id_type=base_sharding.keyspace_id_type, + sharding_column_name='custom_ksid_col') + + # Bootstrap vtgate + + utils.apply_vschema(vschema) + + # Adds alias so vtgate can route to replica/rdonly tablets that are not in the same cell, but same alias + + if use_alias: + utils.run_vtctl(['AddCellsAlias', '-cells', 'test_nj,test_ny','region_east_coast'], auto_log=True) + tablet_types_to_wait='MASTER,REPLICA' + else: + tablet_types_to_wait='MASTER' + + utils.VtGate().start( + tablets=[shard_0_master, shard_1_master], + tablet_types_to_wait=tablet_types_to_wait, + cells_to_watch='test_nj,test_ny', + ) + utils.vtgate.wait_for_endpoints('test_keyspace.-80.master', 1) + utils.vtgate.wait_for_endpoints('test_keyspace.80-.master', 1) + + vtgate_conn = self._get_connection() + result = self._execute_on_tablet_type( + vtgate_conn, + 'master', + 'select count(*) from test_table', {}) + self.assertEqual( + result, + ([(3,)], 1, 0, + [('count(*)', self.int_type)])) + + if use_alias: + vtgate_conn = self._get_connection() + result = self._execute_on_tablet_type( + vtgate_conn, + 'master', + 'select count(*) from test_table', {}) + self.assertEqual( + result, + ([(3,)], 1, 0, + [('count(*)', self.int_type)])) + + vtgate_conn = self._get_connection() + result = self._execute_on_tablet_type( + vtgate_conn, + 'replica', + 'select count(*) from test_table', {}) + self.assertEqual( + result, + ([(3,)], 1, 0, + [('count(*)', self.int_type)])) + + vtgate_conn = self._get_connection() + result = self._execute_on_tablet_type( + vtgate_conn, + 'rdonly', + 'select count(*) from test_table', {}) + self.assertEqual( + result, + ([(3,)], 1, 0, + [('count(*)', self.int_type)])) + else: + vtgate_conn = self._get_connection() + try: + self._execute_on_tablet_type( + vtgate_conn, + 'replica', + 'select count(*) from test_table', {}) + self.fail('Expected execute to fail, did not get error') + except Exception as e: + s = str(e) + self.assertIn('80.replica, no valid tablet: node', s) + + vtgate_conn = self._get_connection() + try: + self._execute_on_tablet_type( + vtgate_conn, + 'rdonly', + 'select count(*) from test_table', {}) + self.fail('Expected execute to fail, did not get error') + except Exception as e: + s = str(e) + self.assertIn('80.rdonly, no valid tablet: node', s) + +if __name__ == '__main__': + utils.main() diff --git a/test/cell_no_aliases.py b/test/cell_no_aliases.py new file mode 100755 index 00000000000..5344cfa0a7d --- /dev/null +++ b/test/cell_no_aliases.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Re-runs cell_aliases.py with no aliases.""" + +import cell_aliases +import utils + +if __name__ == '__main__': + cell_aliases.use_aliases = False + utils.main(cell_aliases) diff --git a/test/config.json b/test/config.json index d629184ec8e..87616c1491f 100644 --- a/test/config.json +++ b/test/config.json @@ -170,6 +170,28 @@ "RetryMax": 0, "Tags": [] }, + "cell_no_aliases": { + "File": "cell_no_aliases.py", + "Args": [], + "Command": [], + "Manual": false, + "Shard": 1, + "RetryMax": 0, + "Tags": [ + "site_test" + ] + }, + "cell_aliases": { + "File": "cell_aliases.py", + "Args": [], + "Command": [], + "Manual": false, + "Shard": 1, + "RetryMax": 0, + "Tags": [ + "site_test" + ] + }, "mysql_server": { "File": "mysql_server_test.py", "Args": [], diff --git a/test/utils.py b/test/utils.py index e134ee4a414..e73dccfdfd2 100644 --- a/test/utils.py +++ b/test/utils.py @@ -552,7 +552,8 @@ def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', - l2vtgates=None): + l2vtgates=None, + cells_to_watch=None): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ @@ -567,7 +568,12 @@ def start(self, cell='test_nj', retry_count=2, '-normalize_queries', '-gateway_implementation', vtgate_gateway_flavor().flavor(), ] - args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) + + if cells_to_watch: + args.extend(vtgate_gateway_flavor().flags(cell=cells_to_watch, tablets=tablets)) + else: + args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) + if l2vtgates: args.extend(['-l2vtgate_addrs', ','.join(l2vtgates)]) if tablet_types_to_wait: diff --git a/test/vertical_split.py b/test/vertical_split.py index 91149160022..7f659d655e5 100755 --- a/test/vertical_split.py +++ b/test/vertical_split.py @@ -420,7 +420,7 @@ def test_vertical_split(self): '--tables', '/moving/,view1', '--chunk_count', '10', '--min_rows_per_chunk', '1', - '--min_healthy_rdonly_tablets', '1', + '--min_healthy_tablets', '1', 'destination_keyspace/0'], auto_log=True) @@ -438,7 +438,7 @@ def test_vertical_split(self): '--tables', '/moving/,view1', '--chunk_count', '10', '--min_rows_per_chunk', '1', - '--min_healthy_rdonly_tablets', '1', + '--min_healthy_tablets', '1', 'destination_keyspace/0'], auto_log=True)