Slack vitess upstream sync 2018 09 06.r0

data/test/vtexplain/multi-output/options-output.txt

@@ -14,7 +14,7 @@ select * from user where id in (1,2,3,4,5,6,7,8)
 1 ks_sharded/c0-: select * from user where id in (4, 6, 7, 8) limit 10001
 
 ----------------------------------------------------------------------
-insert into user (id, name) values(2, 'bob')
+insert into user (id, name) values (2, 'bob')
 
 1 ks_sharded/c0-: begin
 1 ks_sharded/c0-: insert into name_user_map(name, user_id) values ('bob', 2) /* _stream name_user_map (name user_id ) ('Ym9i' 2 ); */ /* vtgate:: keyspace_id:da8a82595aa28154c17717955ffeed8b */

data/test/vtexplain/multi-output/target-output.txt

@@ -0,0 +1,18 @@
+----------------------------------------------------------------------
+select * from user where email='null@void.com'
+
+1 ks_sharded/40-80: select * from user where email = 'null@void.com' limit 10001
+
+----------------------------------------------------------------------
+select * from user where id in (1,2,3,4,5,6,7,8)
+
+1 ks_sharded/40-80: select * from user where id in (1, 2, 3, 4, 5, 6, 7, 8) limit 10001
+
+----------------------------------------------------------------------
+insert into user (id, name) values (2, 'bob')
+
+1 ks_sharded/40-80: begin
+1 ks_sharded/40-80: insert into user(id, name) values (2, 'bob')/* vtgate:: filtered_replication_unfriendly */
+2 ks_sharded/40-80: commit
+
+----------------------------------------------------------------------

data/test/vtexplain/options-queries.sql

@@ -1,3 +1,3 @@
 select * from user where email='null@void.com';
 select * from user where id in (1,2,3,4,5,6,7,8);
-insert into user (id, name) values(2, 'bob');
+insert into user (id, name) values (2, 'bob');

data/test/vtexplain/target-queries.sql

@@ -0,0 +1,8 @@
+/*
+ * These are the same set of queries from "options-queries.sql" but
+ * are run with an explicit shard target of ks_sharded/40-80 which
+ * bypasses the normal v3 routing.
+ */
+select * from user where email='null@void.com';
+select * from user where id in (1,2,3,4,5,6,7,8);
+insert into user (id, name) values (2, 'bob');

data/test/vtgate/filter_cases.txt

@@ -766,4 +766,4 @@
 # but they refer to different things. The first reference is to the outermost query,
 # and the second reference is to the the innermost 'from' subquery.
 "select id2 from user uu where id in (select id from user where id = uu.id and user.col in (select col from (select id from user_extra where user_id = 5) uu where uu.user_id = uu.id))"
-"unsupported: subquery and parent route to different shards"
+"unsupported: UNION or subquery on different shards: vindex values are different"

data/test/vtgate/schema_test.json

@@ -12,7 +12,7 @@
           "owner": "multicolvin"
         },
         "user_md5_index": {
-          "type": "hash_test"
+          "type": "unicode_loose_md5"
         },
         "music_user_map": {
           "type": "lookup_test",
@@ -156,6 +156,9 @@
             }
           ]
         },
+        "pin_test": {
+          "pinned": "80"
+        },
         "weird`name": {
           "column_vindexes": [
             {

data/test/vtgate/select_cases.txt

@@ -254,6 +254,26 @@
   }
 }
 
+# select from pinned table
+"select * from pin_test"
+{
+  "Original": "select * from pin_test",
+  "Instructions": {
+    "Opcode": "SelectEqualUnique",
+    "Keyspace": {
+      "Name": "user",
+      "Sharded": true
+    },
+    "Query": "select * from pin_test",
+    "FieldQuery": "select * from pin_test where 1 != 1",
+    "Vindex": "binary",
+    "Values": [
+      "\ufffd"
+    ]
+  }
+}
+
+
 # select from dual on sharded keyspace
 "select @@session.auto_increment_increment from user.dual"
 {

data/test/vtgate/unsupported_cases.txt

@@ -1,6 +1,6 @@
 # Unions
 "select * from user union select * from user_extra"
-"unsupported: UNION on multi-shard queries"
+"unsupported: UNION or subquery containing multi-shard queries"
 
 # SET
 "set a=1"
@@ -29,11 +29,11 @@
 
 # union operations in subqueries (FROM)
 "select * from (select * from user union all select * from user_extra) as t"
-"unsupported: UNION on multi-shard queries"
+"unsupported: UNION or subquery containing multi-shard queries"
 
 # union operations in subqueries (expressions)
 "select * from user where id in (select * from user union select * from user_extra)"
-"unsupported: UNION on multi-shard queries"
+"unsupported: UNION or subquery containing multi-shard queries"
 
 # subquery with join primitive (expressions)
 "select * from user where id in (select user.id from user join user_extra)"
@@ -77,11 +77,11 @@
 
 # subquery does not depend on unique vindex of outer query
 "select id from user where id in (select user_id from user_extra where user_extra.user_id = user.col)"
-"unsupported: subquery does not depend on scatter outer query"
+"unsupported: UNION or subquery containing multi-shard queries"
 
 # subquery does not depend on scatter outer query
 "select id from user where id in (select user_id from user_extra where user_extra.user_id = 4)"
-"unsupported: subquery does not depend on scatter outer query"
+"unsupported: UNION or subquery containing multi-shard queries"
 
 # subquery depends on a cross-shard subquery
 "select id from (select user.id, user.col from user join user_extra) as t where id in (select t.col from user)"
@@ -105,7 +105,7 @@
 
 # subquery and outer query route to different shards
 "select id from user where id = 5 and id in (select user_id from user_extra where user_extra.user_id = 4)"
-"unsupported: subquery and parent route to different shards"
+"unsupported: UNION or subquery on different shards: vindex values are different"
 
 # last_insert_id for sharded keyspace
 "select last_insert_id() from user"
@@ -433,7 +433,7 @@
 
 # multi-shard union
 "(select id from user union select id from music) union select 1 from dual"
-"unsupported: UNION on multi-shard queries"
+"unsupported: UNION or subquery containing multi-shard queries"
 
 # multi-shard union
 "select 1 from music union (select id from user union all select name from unsharded)"
@@ -445,15 +445,19 @@
 
 # multi-shard union
 "select id from user union all select id from music"
-"unsupported: UNION on multi-shard queries"
+"unsupported: UNION or subquery containing multi-shard queries"
+
+# union with the same target shard because of vindex
+"select * from music where id = 1 union select * from user where id = 1"
+"unsupported: UNION or subquery on different shards: vindexes are different"
 
 # union with different target shards
 "select 1 from music where id = 1 union select 1 from music where id = 2"
-"unsupported: UNION queries with different target shards"
+"unsupported: UNION or subquery on different shards: vindex values are different"
 
 # Union all
 "select col1, col2 from user union all select col1, col2 from user_extra"
-"unsupported: UNION on multi-shard queries"
+"unsupported: UNION or subquery containing multi-shard queries"
 
 "(select user.id, user.name from user join user_extra where user_extra.extra = 'asdf') union select 'b','c' from user"
 "unsupported construct: SELECT of UNION is non-trivial"

doc/GettingStarted.md

@@ -179,45 +179,24 @@ In addition, Vitess requires the software and libraries listed below.
 5.  Run the following commands:
 
     ``` sh
-    brew install go automake libtool python git bison curl wget homebrew/versions/mysql56
+    brew install go automake libtool python git bison curl wget mysql56
     pip install --upgrade pip setuptools
     pip install virtualenv
     pip install MySQL-python
     pip install tox
+
     ```
 
-6.  Install Java runtime from this URL: https://support.apple.com/kb/dl1572?locale=en_US
-    Apple only supports Java 6. If you need to install a newer version, this link might be helpful:
-    [http://osxdaily.com/2015/10/17/how-to-install-java-in-os-x-el-capitan/](http://osxdaily.com/2015/10/17/how-to-install-java-in-os-x-el-capitan/)
-
-7.  The Vitess bootstrap script makes some checks for the go runtime, so it is recommended to have the following
-    commands in your ~/.profile or ~/.bashrc or ~/.zshrc:
+6.  The Vitess bootstrap script makes some checks for the go runtime, so it is recommended to have the following
+    commands in your ~/.profile or ~/.bashrc or ~/.zshrc or ~/.bash_profile:
 
     ``` sh
-    export PATH=/usr/local/opt/go/libexec/bin:$PATH
-    export GOROOT=/usr/local/opt/go/libexec
-    ```
-
-8.  There is a problem with installing the enum34 Python package using pip, so the following file has to be edited:
-    ```
-    /usr/local/opt/python/Frameworks/Python.framework/Versions/2.7/lib/python2.7/distutils/distutils.cfg
-    ```
-
-    and this line:
-
-    ```
-    prefix=/usr/local
+    export PATH="/usr/local/opt/mysql@5.6/bin:$PATH"
+    export PATH=/usr/local/go/bin:$PATH
+    export GOROOT=/usr/local/go
     ```
 
-    has to be commented out:
-
-    ```
-    # prefix=/usr/local
-    ```
-
-    After running the ./bootstrap.sh script from the next step, you can revert the change.
-
-9.  For the Vitess hostname resolving functions to work correctly, a new entry has to be added into the /etc/hosts file
+7.  For the Vitess hostname resolving functions to work correctly, a new entry has to be added into the /etc/hosts file
     with the current LAN IP address of the computer (preferably IPv4) and the current hostname, which you get by
     typing the 'hostname' command in the terminal.
 
@@ -232,38 +211,40 @@ In addition, Vitess requires the software and libraries listed below.
 
 1.  Navigate to the directory where you want to download the Vitess
     source code and clone the Vitess Github repo. After doing so,
-    navigate to the `src/vitess.io/vitess` directory.
+    navigate to the `src/vitess.io/vitess` directory. For go to work 
+    correctly, you should create a symbolic link to this inide your ${HOME}/go/src
 
     ``` sh
     cd $WORKSPACE
     git clone https://github.com/vitessio/vitess.git \
         src/vitess.io/vitess
-    cd src/vitess.io/vitess
+    ln -s src/vitess.io ${HOME}/go/src/vitess.io
+    cd ${HOME}/go/src/vitess.io/vitess
     ```
 
 1.  Set the `MYSQL_FLAVOR` environment variable. Choose the appropriate
     value for your database. This value is case-sensitive.
 
     ``` sh
-    export MYSQL_FLAVOR=MariaDB
+    # export MYSQL_FLAVOR=MariaDB
     # or (mandatory for OS X)
-    # export MYSQL_FLAVOR=MySQL56
+    export MYSQL_FLAVOR=MySQL56
     ```
 
 1.  If your selected database installed in a location other than `/usr/bin`,
     set the `VT_MYSQL_ROOT` variable to the root directory of your
-    MariaDB installation. For example, if MariaDB is installed in
+    MariaDB installation. For example, if mysql is installed in
     `/usr/local/mysql`, run the following command.
 
     ``` sh
-    export VT_MYSQL_ROOT=/usr/local/mysql
+    # export VT_MYSQL_ROOT=/usr/local/mysql
 
     # on OS X, this is the correct value:
-    # export VT_MYSQL_ROOT=/usr/local/opt/mysql56
+    export VT_MYSQL_ROOT=/usr/local/opt/mysql@5.6
     ```
 
     Note that the command indicates that the `mysql` executable should
-    be found at `/usr/local/mysql/bin/mysql`.
+    be found at `/usr/local/opt/mysql@5.6/bin/mysql`.
 
 1.  Run `mysqld --version` and confirm that you
     are running the correct version of MariaDB or MySQL. The value should

doc/GettingStartedKubernetes.md

@@ -278,7 +278,7 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl
 1.  **Access vtctld web UI**
 
     To access vtctld from outside Kubernetes, use [kubectl proxy]
-    (http://kubernetes.io/v1.1/docs/user-guide/kubectl/kubectl_proxy.html)
+    (https://kubernetes.io/docs/tasks/access-kubernetes-api/http-proxy-access-api/)
     to create an authenticated tunnel on your workstation:
 
     **Note:** The proxy command runs in the foreground,
@@ -292,13 +292,13 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl
 
     You can then load the vtctld web UI on `localhost`:
 
-    http://localhost:8001/api/v1/proxy/namespaces/default/services/vtctld:web/
+    http://localhost:8001/api/v1/namespaces/default/services/vtctld:web/proxy
 
     You can also use this proxy to access the [Kubernetes Dashboard]
-    (http://kubernetes.io/v1.1/docs/user-guide/ui.html),
+    (https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/),
     where you can monitor nodes, pods, and services:
 
-    http://localhost:8001/ui
+    http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/.
 
 1.  **Use vtctlclient to send commands to vtctld**
 

doc/VtExplain.md

@@ -94,42 +94,50 @@ vtexplain -shards 8 -vschema-file /tmp/vschema.json -schema-file /tmp/schema.sql
 ----------------------------------------------------------------------
 SELECT * from users
 
-[mainkeyspace/c0-]:
-select * from users limit 10001
+1 mainkeyspace/-20: select * from users limit 10001
+1 mainkeyspace/20-40: select * from users limit 10001
+1 mainkeyspace/40-60: select * from users limit 10001
+1 mainkeyspace/60-80: select * from users limit 10001
+1 mainkeyspace/80-a0: select * from users limit 10001
+1 mainkeyspace/a0-c0: select * from users limit 10001
+1 mainkeyspace/c0-e0: select * from users limit 10001
+1 mainkeyspace/e0-: select * from users limit 10001
 
-[mainkeyspace/-40]:
-select * from users limit 10001
+----------------------------------------------------------------------
+```
 
-[mainkeyspace/40-80]:
-select * from users limit 10001
+The output shows the sequence of queries run.
 
-[mainkeyspace/80-c0]:
-select * from users limit 10001
+In this case, the query planner is a scatter query to all shards, and each line shows:
 
-----------------------------------------------------------------------
-```
+(a) the logical sequence of the query
+(b) the keyspace/shard
+(c) the query that was executed
+
+The fact that each query runs at time `1` shows that vitess executes these in parallel, and the `limit 10001` is automatically added as a protection against large results.
 
 **Insert:**
 
 ```bash
-vtexplain -shards 1024 -vschema-file /tmp/vschema.json -schema-file /tmp/schema.sql -replication-mode "ROW" -output-mode text -sql "INSERT INTO users (user_id, name) VALUES(1, 'john')"
+vtexplain -shards 128 -vschema-file /tmp/vschema.json -schema-file /tmp/schema.sql -replication-mode "ROW" -output-mode text -sql "INSERT INTO users (user_id, name) VALUES(1, 'john')"
 
 ----------------------------------------------------------------------
 INSERT INTO users (user_id, name) VALUES(1, 'john')
 
-[mainkeyspace/22c0-2300]:
-begin
-insert into users_name_idx(name, user_id) values ('john', 1)
-commit
-
-[mainkeyspace/1640-1680]:
-begin
-insert into users(user_id, name) values (1, 'john')
-commit
+1 mainkeyspace/22-24: begin
+1 mainkeyspace/22-24: insert into users_name_idx(name, user_id) values ('john', 1) /* vtgate:: keyspace_id:22c0c31d7a0b489a16332a5b32b028bc */
+2 mainkeyspace/16-18: begin
+2 mainkeyspace/16-18: insert into users(user_id, name) values (1, 'john') /* vtgate:: keyspace_id:166b40b44aba4bd6 */
+3 mainkeyspace/22-24: commit
+4 mainkeyspace/16-18: commit
 
 ----------------------------------------------------------------------
 ```
 
+This example shows how Vitess handles an insert into a table with a secondary lookup vindex.
+
+First, at time `1`, a transaction is opened on one shard to insert the row into the `users_name_idx` table. Then at time `2` a second transaction is opened on another shard with the actual insert into the `users` table, and finally each transaction is committed at time `3` and `4`.
+
 **Configuration Options**
 
 The `--shards` option specifies the number of shards to simulate. vtexplain will always allocate an evenly divided key range to each.

examples/compose/README.md

@@ -10,13 +10,20 @@ To start Consul(which saves the topology config), vtctld, vtgate and a few vttab
 ```
 vitess/examples/compose$ docker-compose up -d
 ```
+Note that the vtgate container will likely fail to start.
 
 ### Load the schema
 We need to create a few tables into our new cluster. To do that, we can run the `ApplySchema` command.
 ```
 vitess/examples/compose$ ./lvtctl.sh ApplySchema -sql "$(cat create_test_table.sql)" test_keyspace
 ```
 
+### Complete starting the cluster
+Now that schema has been loaded a second start will bring vtgate up as well.
+```
+vitess/examples/compose$ docker-compose up -d
+```
+
 ### Run the client to insert and read some data
 This will build and run the `client.go` file. It will insert and read data from the master and from the replica.
 ```