diff --git a/.gitignore b/.gitignore
index 9a4aec200..1b01c4be9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,4 +6,7 @@ resources/
# Link checker
bin/
tmp/
-.idea
\ No newline at end of file
+.idea
+
+# File generated when running npm install
+package-lock.json
\ No newline at end of file
diff --git a/README.md b/README.md
index cb78140b6..c7ff483ee 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,9 @@ This repo houses the assets used to build the website at https://vitess.io.
## Running the site locally
-To run the website locally, you need to have the [Hugo](https://gohugo.io) static site generator installed (installation instructions [here](https://gohugo.io/getting-started/installing/)). Once Hugo is installed run the following:
+To run the website locally, you need to have the [Hugo](https://gohugo.io) static site generator installed (installation instructions [here](https://gohugo.io/getting-started/installing/)). Installing the Hugo version in [netlify.toml](./netlify.toml) is recommended.
+
+Once Hugo is installed run the following:
```bash
hugo server --buildDrafts --buildFuture
diff --git a/assets/sass/docs.sass b/assets/sass/docs.sass
index 489c6c56f..1c5f7c6c3 100644
--- a/assets/sass/docs.sass
+++ b/assets/sass/docs.sass
@@ -1,5 +1,4 @@
$docs-sidebar-logo-width: 30%
-$docs-sidebar-margin: 1.75rem
$docs-sidebar-header-margin-bottom: 2rem
$tan: #f5993a
@@ -12,156 +11,156 @@ html, body
=h-center
justify-content: center
+// Bit of a hacky full-width container override for the docs page,
+// since bulma.io doesn't appear to have a helper for this.
+.navbar-container-docs
+ max-width: 100%
+ padding: 0 8px 0 24px
+
.docs-wrapper
+flex
flex-direction: row
- height: 100vh
+ height: calc(100vh - #{$navbar-height})
align-items: stretch
- .docs-navbar
- position: sticky
- top: 0
+.docs-navbar
+ position: sticky
+ top: 0
- .docs-sidebar
- +flex
- +h-center
- background-color: $dark
- flex: 0 0 15%
- border-right: 1px solid lighten($dark, 10%)
- overflow: scroll
+.docs-sidebar
+ +flex
+ +h-center
+ background-color: $dark
+ flex: 0 0 15%
+ border-right: 1px solid lighten($dark, 10%)
+ overflow: auto
- .docs-menu
- width: 100%
- margin: $docs-sidebar-margin
+.docs-menu
+ width: 100%
+ margin: 24px 0 24px 0
- .docs-menu-header
- margin-bottom: $docs-sidebar-header-margin-bottom
+.docs-menu-header
+ margin-bottom: $docs-sidebar-header-margin-bottom
- .docs-menu-logo
- width: $docs-sidebar-logo-width
- margin: 0 auto
- display: block
+.docs-menu-logo
+ margin: 0 auto
+ display: block
- .docs-menu-bar
- padding-bottom: 5rem
+.docs-menu > .docs-navlist > li > a
+ font-size: 1.2rem
- .docs-section
- & + .docs-section
- margin-top: .75rem
+.docs-navlist ul > li
+ border-left: solid 1px rgba(255, 255, 255, 0.1)
+ margin-left: 16px
- .docs-section-title
- font-size: 1.2rem
- color: $grey-lighter
- line-height: 100%
+.docs-navlist li
+ position: relative
- &.is-active
- color: $primary
+.docs-navlist li a
+ color: $grey-lighter
+ display: block
+ font-size: 0.9rem
+ padding: 8px 24px 8px 16px
- .docs-section-list
- margin-left: .3rem
- border-left: 1px solid $grey
+ &:hover
+ background: rgba(255, 255, 255, 0.05)
- ul
- margin-top: .75rem
+.docs-navlist li.active > a
+ color: $tan
- li
- font-size: 1rem
- margin-left: 1rem
- line-height: 120%
+.navlist-caret
+ height: 10px
+ margin-top: -5px
+ position: absolute
+ right: 16px
+ top: 22px
+ fill: rgba(255, 255, 255, 0.25)
- a
- color: $grey-lighter
+li.expanded > a > .navlist-caret
+ transform: rotate(90deg)
- & + li
- margin-top: .8rem
+.docs-article
+ flex: 1 1 auto
+ overflow: auto
- &.is-active
- a
- color: $tan
+ .docs-content
+ margin: 2rem 2.5rem
- .docs-article
- flex: 1 1 auto
- overflow: auto
+ +touch
+ margin: 1.5rem
- .docs-content
- margin: 2rem 2.5rem
+ padding-bottom: 5rem
- +touch
- margin: 1.5rem
+ .content
+ +desktop
+ width: 80%
- padding-bottom: 5rem
+ p a, li a
+ font-weight: 700
- .content
- +desktop
- width: 80%
+ li .highlight, ol li
+ margin-bottom: 1rem
- p a, li a
- font-weight: 700
+ h1
+ font-size: 1.802rem
+ h2
+ font-size: 1.602rem
+ h3
+ font-size: 1.424rem
+ h4
+ font-size: 1.266rem
+ h5
+ font-size: 1.125rem
+ h2, h3, h4, h5, h6
+ color: $vitess-orange
- li .highlight, ol li
- margin-bottom: 1rem
+ .docs-header
+ padding-bottom: 1rem
+ margin-bottom: 2rem
- h1
- font-size: 1.802rem
- h2
- font-size: 1.602rem
- h3
- font-size: 1.424rem
- h4
- font-size: 1.266rem
- h5
- font-size: 1.125rem
- h2, h3, h4, h5, h6
- color: $vitess-orange
+ .card-content
+ padding: 0 0 1.5rem 0
- .docs-header
- padding-bottom: 1rem
- margin-bottom: 2rem
- border-bottom: 1px solid $grey-lighter
+.docs-toc
+ flex: 0 0 15%
+ background-color: $white-bis
+ overflow: auto
+ border-left: 1px solid $grey-light
- .card-content
- padding: 0 0 1.5rem 0
+ &-container
+ margin: 1rem
- .docs-toc
- flex: 0 0 15%
- background-color: $white-bis
- overflow: auto
- border-left: 1px solid $grey-light
+ &-title
+ color: $dark
+ font-size: 1.25rem
- &-container
- margin: 1rem
+ &-menu
+ margin: 1.5rem 0
- &-title
+ a
color: $dark
- font-size: 1.25rem
-
- &-menu
- margin: 1.5rem 0
-
- a
- color: $dark
- &:hover
- color: $primary
+ &:hover
+ color: $primary
- #TableOfContents
- ul
- li
- font-size: 1rem
- line-height: 1.2rem
+ #TableOfContents
+ ul
+ li
+ font-size: 1rem
+ line-height: 1.2rem
- & + li
- margin-top: .75rem
+ & + li
+ margin-top: .75rem
- ul
- margin: .5rem 0 0 1rem
+ ul
+ margin: .5rem 0 0 1rem
- li
- font-size: 0.8rem
+ li
+ font-size: 0.8rem
- & + li
- margin-top: .75rem
- line-height: 1.2rem
+ & + li
+ margin-top: .75rem
+ line-height: 1.2rem
@@ -171,4 +170,3 @@ html, body
margin-left: 30px;
background-color: #f7f7f7;
margin-bottom: 10px;
-
diff --git a/assets/sass/style.sass b/assets/sass/style.sass
index b6c2310fc..9d0ee13a7 100644
--- a/assets/sass/style.sass
+++ b/assets/sass/style.sass
@@ -53,11 +53,12 @@ figure
.has-extra-padding
padding: 2.5rem
-.docs-article ul li
+.docs-article .content ul li
margin-left: -.5rem
list-style-type: square
.is-blog-container
+ max-width: 100%
margin: -12rem auto 0 auto
.is-blog-card
@@ -94,6 +95,8 @@ figure
font-style: normal
font-weight: 700
+.is-blog-container .media-content
+ max-width: 100%
.toc-column
@@ -147,8 +150,8 @@ figure
justify-content: right
.is-user-logo
- max-height: 1.5rem
- margin: 1.25rem
+ max-height: 1.7rem
+ margin: 1.1rem
=logo($touch, $tablet)
+touch
@@ -163,7 +166,7 @@ figure
+logo(40%, 30%)
.is-cncf-logo
- +logo(60%, 35%)
+ +logo(55%, 30%)
.is-footer-logo
+logo(30%, 10%)
@@ -190,3 +193,45 @@ figure
&.is-active
position: fixed
++until($navbar-breakpoint)
+ #search-bar
+ display: block
+
+ .algolia-autocomplete
+ // Fixes for the DocSearch widget to prevent clipping on mobile.
+ display: block !important
+
+ .ds-dropdown-menu
+ max-width: 100vw !important
+ min-width: fit-content !important
+
++from($navbar-breakpoint)
+ // On mobile (and by default), the search input is displayed at the top of the menu,
+ // whereas on desktop, it's displayed at the right side of the menu bar (i.e., flex-end)
+ .navbar-menu
+ flex-direction: row-reverse
+
+ // Cuddle up the search input + social icons for the full-width navbar
+ .navbar-item-search
+ padding-left: 0
+
+.header-link
+ color: $dark
+ opacity: 0.3
+
+ &:hover
+ opacity: 1
+
+// When jumping to anchor links, we need to account for the height of the sticky navbar.
+// Otherwise, the anchored header will be occluded by the navbar.
+// See https://css-tricks.com/hash-tag-links-padding/
+$navbar-anchor-offset: calc(#{$navbar-height} + 16px)
+
+h1, h2, h3, h4, h5, h6
+ &.anchored-header[id]:before
+ display: block
+ content: ' '
+ margin-top: calc(-1 * #{$navbar-anchor-offset})
+ height: $navbar-anchor-offset
+ visibility: hidden
+ pointer-events: none
diff --git a/config.toml b/config.toml
index 9ecf8a0b1..2570ab079 100644
--- a/config.toml
+++ b/config.toml
@@ -2,6 +2,7 @@ baseURL = "https://vitess.io"
canonifyurls = true
disableKinds = ["taxonomy", "taxonomyTerm"]
googleAnalytics = "UA-163836834-1"
+enableRobotsTXT = true
# Syntax highlighting settings
pygmentsCodeFences = true
@@ -21,6 +22,11 @@ stackoverflow = "vitess"
[params.blog]
subtitle = "Updates and insights from the **Vitess** team."
+[sitemap]
+changefreq = "monthly"
+filename = "sitemap.xml"
+priority = 0.5
+
# Language settings
DefaultContentLanguage = "en"
defaultContentLanguageInSubdir = true
diff --git a/content/en/blog/2015-10-06-cloud-native-mysql-sharding-with-vitess-and-kubernetes.md b/content/en/blog/2015-10-06-cloud-native-mysql-sharding-with-vitess-and-kubernetes.md
index 48f742a34..0d2325ec5 100644
--- a/content/en/blog/2015-10-06-cloud-native-mysql-sharding-with-vitess-and-kubernetes.md
+++ b/content/en/blog/2015-10-06-cloud-native-mysql-sharding-with-vitess-and-kubernetes.md
@@ -21,9 +21,9 @@ of [Vitess v2.0.0](https://github.com/youtube/vitess/releases). Some highlights
* Java and Go clients use the new HTTP/2-based [gRPC](http://www.grpc.io/) framework.
* Can now run on top of MySQL 5.6, in addition to MariaDB 10.0.
* New administrative dashboard built on AngularJS.
-* Built-in backup/restore](http://vitess.io/user-guide/backup-and-restore.html), designed to plug into blob stores like [Google Cloud Storage](https://cloud.google.com/storage/).
-* GTID-based [reparenting](http://vitess.io/user-guide/reparenting.html) for reversible, routine failovers.
-* Simpler [schema changes](http://vitess.io/user-guide/schema-management.html).
+* [Built-in backup/restore](https://vitess.io/docs/user-guides/operating-vitess/backup-and-restore/) designed to plug into blob stores like [Google Cloud Storage](https://cloud.google.com/storage/).
+* GTID-based [reparenting](https://vitess.io/docs/user-guides/configuration-advanced/reparenting/) for reversible, routine failovers.
+* Simpler [schema changes](https://vitess.io/docs/reference/features/schema-management/).
We've also been hard at work adding lots more [documentation](http://vitess.io/user-guide/introduction.html). In particular, the rest of this post will explore one of our new walkthroughs that demonstrates transparent [resharding](http://vitess.io/user-guide/sharding.html#resharding) of a live database - that is, changing the number of shards without any code changes or noticeable downtime for the application.
@@ -38,30 +38,22 @@ values, which we call [keyspace IDs](http://vitess.io/overview/concepts.html#key
**Transparent Resharding**
-If you want to follow along with the new [resharding walkthrough](http://vitess.io/user-guide/sharding-kubernetes.html), you'll need to first bring up the cluster as described in the [unsharded guide](http://vitess.io/getting-started/). Both guides use the same [sample app](https://github.com/youtube/vitess/tree/master/examples/kubernetes/guestbook), which is a Guestbook that supports multiple, numbered pages.
-
-
+If you want to follow along with the new [resharding walkthrough](http://vitess.io/user-guide/sharding-kubernetes.html), you'll need to first bring up the cluster as described in the [unsharded guide](http://vitess.io/getting-started/).
In the [sample app code](https://github.com/youtube/vitess/blob/master/examples/kubernetes/guestbook/main.py), you'll see a `get\_keyspace\_id()` function that transforms a given page number to the set of all 8-byte sequences, establishing the mapping we need for consistent hashing. In the unsharded case, these values are stored but not used. When we introduce sharding, page numbers will be evenly distributed (on average) across all the shards we create, allowing the app to scale to support arbitrary amounts of pages.
Before resharding, you'll see a single [custom shard](http://vitess.io/user-guide/sharding.html#custom-sharding) named "0" in the Vitess dashboard. This is what an unsharded [keyspace](http://vitess.io/overview/concepts.html#keyspace) looks like.
-
-
As you begin the [resharding walkthrough](http://vitess.io/user-guide/sharding-kubernetes.html),you'll bring up two new shards for the same keyspace. During resharding, the new shards will run alongside the old one, but they'll remain idle (Vitess will not route any app traffic to them) until you're ready to migrate. In the dashboard, you'll see all three shards, but only shard "0" is currently active.
-
-
Next, you'll run a few Vitess commands to [copy the schema and data](http://vitess.io/user-guide/sharding-kubernetes.html#copy-data-from-original-shard) from the original shard. The key to live migration is that once the initial snapshot copy is done, Vitess will automatically begin replicating fresh updates on the original shard to the new shards. We call this [filtered replication](http://vitess.io/user-guide/sharding.html#filtered-replication), since it distributes DMLs only to the shards to which they apply. Vitess also includes tools that compare the original and copied data sets, row-by-row, to [verify data
integrity](http://vitess.io/user-guide/sharding-kubernetes.html#check-copied-data-integrity).
Once you've verified the copy, and filtered replication has caught up to real-time updates, you can run the [migrate command](http://vitess.io/user-guide/sharding-kubernetes.html#switch-over-to-the-new-shards) which tells Vitess to atomically shift app traffic from the old shards to the new ones. It does this by disabling writes on the old masters, waiting for the new masters to receive the last events over filtered replication, and then enabling writes on the new masters. Since the process is automated, this typically only causes about a second of write unavailability.
-Now you can[tear down the old shard](http://vitess.io/user-guide/sharding-kubernetes.html#remove-the-original-shard),
+Now you can [tear down the old shard](http://vitess.io/user-guide/sharding-kubernetes.html#remove-the-original-shard),
and verify that only the new ones show up in the dashboard.
-
-
Note that we never had to tell the app that we were changing from one shard to two. The resharding process was completely transparent to the app, since Vitess automatically reroutes queries on-the-fly as the migration progresses.
At YouTube, we've used Vitess to transparently reshard (both [horizontally and vertically](http://vitess.io/user-guide/sharding.html#supported-operations)) nearly all of our MySQL databases within the last year alone, and we have still more on the horizon as we continue to grow. See the full [walkthrough instructions](http://vitess.io/user-guide/sharding-kubernetes.html) if you want to try it out for yourself.
@@ -73,14 +65,10 @@ The promise of sharding is that it allows you to scale write throughput linearly
Below you can see preliminary results for scaling write throughput by adding more shards in Vitess running on [Google Container Engine](https://cloud.google.com/container-engine/). For this benchmark, we pointed YCSB at the [load balancer](http://kubernetes.io/v1.0/docs/user-guide/services.html#type-loadbalancer) for our Vitess cluster and told it to send a lot of INSERT statements.
Vitess took care of routing statements to the various shards.
-[](../images/2015-10-06-cloud-native-mysql-sharding-with-vitess-and-kubernetes-vitess%2B3.png)
-
The max throughput (QPS) for a given number of shards is the point at which round-trip write latency became degraded, which we define as >15ms on average or >50ms for the worst 1% of queries (99th percentile).
We also ran YCSB's "read mostly" workload (95% reads, 5% writes) to show how Vitess can scale read traffic by adding replicas. The max throughput here is the point at which round-trip read latency became degraded, which we define as >5ms on average or >20ms for the worst 1% of queries.
-
-
There's still a lot of room to improve the benchmarks (for example, by tuning the performance of MySQL itself). However, these preliminary results show that the returns don't diminish as you scale. And since you're scaling horizontally, you're not limited by the size of a single machine.
**Conclusion**
diff --git a/content/en/blog/2017-09-18-custom-sharding-with-vitess.md b/content/en/blog/2017-09-18-custom-sharding-with-vitess.md
index 4dc67c2cb..7d77f60c2 100644
--- a/content/en/blog/2017-09-18-custom-sharding-with-vitess.md
+++ b/content/en/blog/2017-09-18-custom-sharding-with-vitess.md
@@ -119,7 +119,7 @@ The advantage of this approach is that it later allows us to change the vindex t
The next obvious question is: why mod at all? What if we just used `ReverseBits(user_id)`? It turns out that this would also work. There was really no need to perform the mod in the first place. Once you’ve transitioned to using `ReverseBits`, you can shard at will from any number to any number. Over time, you can forget that you ever used mod-based sharding.
-The sample code for the above Custom Vindex is [available here](https://gist.github.com/sougou/96e40aa54526447ae0b24d50ae8ea4a8). This Vindex is handy enough that we will look at adding it to the Vitess list of predefined vindexes.
+This is now available as a predefined Vindex as [reverse_bits](https://github.com/vitessio/vitess/blob/master/go/vt/vtgate/vindexes/reverse_bits.go).
Can you think of other ways to perform such migrations? Join us on our Slack channel to share your ideas. You can send an email to [vitess@googlegroups.com](mailto:vitess@googlegroups.com)
to request an invite.
diff --git a/content/en/blog/2020-04-29-announcing-vitess-6.md b/content/en/blog/2020-04-29-announcing-vitess-6.md
index 04182e07b..27c4211c8 100644
--- a/content/en/blog/2020-04-29-announcing-vitess-6.md
+++ b/content/en/blog/2020-04-29-announcing-vitess-6.md
@@ -21,7 +21,7 @@ The Helm charts now default to using Kubernetes as the [Topology Service](https:
This change also unlocked the adoption of Helm 3 and support for a wider range of Kubernetes versions, making installing Vitess much easier.
### General Availability of VReplication-based Workflows
-While VReplication made its appearance in Vitess 4, it has now been promoted from experimental to general availability and the documentation now points to [MoveTables](https://vitess.io/docs/user-guides/move-tables/) and [Resharding](https://vitess.io/docs/user-guides/resharding/).
+While VReplication made its appearance in Vitess 4, it has now been promoted from experimental to general availability and the documentation now points to [MoveTables](https://vitess.io/docs/user-guides/migration/move-tables/) and [Resharding](https://vitess.io/docs/user-guides/configuration-advanced/resharding/).
These workflows require significantly fewer steps than their predecessors (Vertical Split Clone and Horizontal Sharding), of which we intend to deprecate at some point in the future.
diff --git a/content/en/blog/2020-07-28-announcing-vitess-7.md b/content/en/blog/2020-07-28-announcing-vitess-7.md
index 71630e7e8..f56735530 100644
--- a/content/en/blog/2020-07-28-announcing-vitess-7.md
+++ b/content/en/blog/2020-07-28-announcing-vitess-7.md
@@ -4,6 +4,7 @@ date: 2020-07-28
slug: '2020-07-28-announcing-vitess-7'
tags: ['release']
title: 'Announcing Vitess 7'
+description: 'We are pleased to announce the general availability of Vitess 7. Highlights include replica transactions, savepoint support, and per-session system variables.'
---
On behalf of the Vitess maintainers team, I am pleased to announce the general availability of Vitess 7.
diff --git a/content/en/blog/2020-10-27-announcing-vitess-8.md b/content/en/blog/2020-10-27-announcing-vitess-8.md
new file mode 100644
index 000000000..d7259de52
--- /dev/null
+++ b/content/en/blog/2020-10-27-announcing-vitess-8.md
@@ -0,0 +1,60 @@
+---
+author: 'Alkin Tezuysal'
+date: 2020-10-27
+slug: '2020-10-27-announcing-vitess-8'
+tags: ['release']
+title: 'Announcing Vitess 8'
+description: 'We are pleased to announce the general availability of Vitess 8. In this release, we have continued to make important improvements to the Vitess project with over 200 PRs in several areas.'
+---
+On behalf of the Vitess maintainers team, I am pleased to announce the general availability of [Vitess 8](https://github.com/vitessio/vitess/releases/tag/v8.0.0).
+
+## Major Themes
+In this release, we have continued to make important improvements to the Vitess project with over 200 PRs in several areas. Some of the major bug fixes and changes in behaviors are documented in the [Release Notes](https://github.com/vitessio/vitess/blob/master/doc/releasenotes/8_0_0_release_notes.md). Please read them carefully and report any issues via GitHub. We would like to highlight the following themes for this release.
+
+### Compatibility (MySQL, frameworks)
+
+Our ongoing work to make sure that Vitess accepts all queries that MySQL accepts. In particular, work has focused on SET and information_schema queries. Reserved connections are still not on by default, and you might need to enable it to see all queries and frameworks well supported.
+
+We are proud to announce that we have initial support for:
+* PHP
+ * WordPress
+ * Mysqli
+* JavaScript
+ * TypeORM
+ * Sequelize
+* Python
+ * Django
+ * PyMySQL
+ * SQLAlchemy
+* Ruby
+ * Rails/ActiveRecord
+* Java
+ * JDBC
+ * Hibernate
+* Rust
+ * MySQL
+ * mysql_async
+ * SQLx
+* Tooling
+ * MySQL Workbench
+ * Mycli
+
+### Migration
+Performance and error metrics and improved logging related to VReplication workflows have been added for more visibility into operational issues. Additional vtctld commands VExec and Workflow allow easier inspection and manipulation of VReplication streams.
+
+The VStream API was enhanced to provide more information for integration with change data capture platforms: the Debezium Vitess adapter uses this capability.
+
+We have incorporated several small feature enhancements and bug-fixes based on the increased traction that VReplication saw both among early adopters and large production setups.
+
+### Usability
+Ease of usability and accessibility are very important for the Vitess community. Usability improvements were another highlight received from the community.
+
+### Innovation
+We continue to add integration of popular open-source tools and utilities on top of the Vitess’s dynamic framework. There are a few of these in this release we would like to highlight.
+
+* VTorc : Integration of Orchestrator has continued and finally became part of Vitess. This proven open-source tool which has been the de-facto solution for MySQL failover mechanisms is now built into Vitess. Support is experimental in 8.0 and we will continue to harden it in future releases.
+* [Online Schema Changes](https://vitess.io/docs/user-guides/schema-changes/): Understanding the ALTER TABLE problem and coming up with a solution using proven tools was our goal to achieve this release. We’re able to integrate both pt-online-schema-change and gh-ost to overcome major limitations for schema migrations.
+
+There is a shortlist of incompatible changes in this release. We encourage you to spend a moment reading the release notes.
+
+Please download [Vitess 8](https://github.com/vitessio/vitess/releases/tag/v8.0.0) and try it out!
diff --git a/content/en/blog/2020-11-03-streaming-vitess-at-bolt.md b/content/en/blog/2020-11-03-streaming-vitess-at-bolt.md
new file mode 100644
index 000000000..f0626fb60
--- /dev/null
+++ b/content/en/blog/2020-11-03-streaming-vitess-at-bolt.md
@@ -0,0 +1,137 @@
+---
+author: 'Kewei Shang'
+date: 2020-11-03
+slug: '2020-11-03-streaming-vitess-at-bolt'
+tags: ['Vitess','MySQL','Debezium','Kafka','CDC','Change Data Capture','Apache']
+title: 'Streaming Vitess at Bolt'
+---
+
+
+Previously posted on [link](https://medium.com/bolt-labs/streaming-vitess-at-bolt-f8ea93211c3f) at Nov 3, 2020.
+
+Traditionally, MySQL has been used to power most of the backend services at Bolt. We've designed our schemas in a way that they're sharded into different MySQL clusters. Each MySQL cluster contains a subset of data and consists of one primary and multiple replication nodes.
+
+Once data is persisted to the database, we use the [Debezium MySQL Connector](https://debezium.io/documentation/reference/connectors/mysql.html) to [capture data change](https://www.confluent.io/blog/how-bolt-adopted-cdc-with-confluent-for-real-time-data-and-analytics/) events and send them to Kafka. This gives us an easy and reliable way to communicate changes between back-end microservices.
+
+### Vitess at Bolt
+Bolt has grown considerably over the past few years, and so did the volume of data written to MySQL. Manual database sharding has become quite an expensive and long-lasting process prone to errors. So we started to evaluate more scalable databases, one of which is [Vitess](https://vitess.io/). Vitess is an open-source database clustering system that is based on MySQL and provides horizontal scalability for it. Originated and battle-tested at YouTube, it was later open-sourced and is used by companies like Slack, Github, JD.com to power their backend storage. It combines important MySQL features with the scalability of a NoSQL database.
+
+One of the most important features that Vitess provides is its built-in sharding. It allows the database to grow horizontally by adding new shards in a way that is transparent to back-end application logic. To your application, Vitess appears like a giant single database, but in fact data is partitioned into multiple physical shards behind the scenes. For any table, an arbitrary column can be chosen as the sharding key, and all inserts and updates will be seamlessly directed to a proper shard by Vitess itself.
+
+Figure 1 below illustrates how back-end services interact with Vitess. At a high level, services connect to stateless VTGate instances through a load balancer. Each VTGate has the Vitess cluster's topology cached in its memory and redirects queries to the correct shards and the correct VTTablet (and its underlying MySQL instance) within the shards. More on VTTablet is written below.
+
+
+
+Figure 1. Vitess architecture. Reference: https://www.planetscale.com/vitess
+
+Other useful features provided by Vitess are:
+* Failover (a.k.a. Reparenting) is easy and transparent for clients. Clients only talk to a VTGate who takes care of failover and service discovery of the new primary transparently.
+* It automatically rewrites "problematic" queries that could potentially cause database performance degradation.
+* It has a caching mechanism that prevents duplicate queries to reach the underlying MySQL database simultaneously. Only one query will reach the database and its result will be cached and returned to answer duplicate queries.
+* It has its connection pool and eliminates the high-memory overhead of MySQL connections. As a result, it can easily handle thousands of connections at the same time.
+* Connection timeout and transaction timeout can be configured.
+* It has minimal downtime when doing [resharding](https://vitess.io/docs/user-guides/configuration-advanced/resharding/) operations.
+* Its VStream feature can be used by downstream CDC applications to read change events from Vitess.
+### Streaming Vitess Options
+The ability to capture data changes and publish them to Apache Kafka was one of the requirements for adopting Vitess at Bolt. There were several different options we've considered.
+#### Option 1: Using Debezium MySQL Connector
+Applications connect to Vitess VTGate to send queries. VTGate supports the MySQL protocol and has a SQL parser. You can use any MySQL client (e.g. JDBC) to connect to VTGate, which redirects your query to the correct shard and returns the result to your client.
+
+However, VTGate is not equal to a MySQL instance, it is rather a stateless proxy to various MySQL instances. For the MySQL connector to receive change events, the Debezium MySQL connector needs to connect to a real MySQL instance. To make it more obvious, VTGate also has some known [compatibility](https://vitess.io/docs/reference/compatibility/mysql-compatibility/) issues, which makes connecting to VTGate different from MySQL.
+
+Another option is to use the Debezium MySQL Connector to connect directly to the underlying MySQL instances of different shards. It has its advantages and disadvantages.
+
+One advantage is that for an unsharded keyspace (Vitess's terminology for a database), the MySQL Connector can continue to work correctly and we don't need to include additional logic or specific implementation. It should just work fine.
+
+One of the biggest disadvantages is that resharding operations would become more complex. For example, the GTID of the original MySQL instance would change when resharded, and the MySQL connector depends on the GTID to work correctly. We also believe that having the MySQL connector connected directly to each underlying MySQL instance defies the purpose of Vitess's operational simplicity as a new connector has to be added (or removed) each time resharding is done. Not to mention that such an operation would lead to data duplication inside Kafka brokers.
+#### Option 2: Using JDBC Source Connector
+We've also considered using the [JDBC Source Connector](https://docs.confluent.io/current/connect/kafka-connect-jdbc/source-connector/index.html). It allows sourcing data from any relational databases that support the JDBC driver into Kafka. Therefore, it is compatible with Vitess VTGate. It has its advantages and disadvantages as well.
+
+Advantages:
+* It is compatible with VTGate.
+* It handles Vitess resharding operation better. During resharding operation, reads are simply automatically redirected (by VTGate) to the target shards. It won't generate any duplicates or lose any data.
+
+Disadvantages:
+* It is poll-based, meaning that the connector polls the database for new change events on a defined interval (typically every few seconds). This means that we would have a much higher latency, compared to the Debezium MySQL Connector.
+* Its offsets are managed by either the table's incremental primary key or one of the table's timestamp columns. If we use the timestamp column for offset, we'd have to create a secondary-index of the timestamp column for each table. This adds more constraints on our backend services. If we use the incremental primary key, we would miss the change events for row-updates because the primary key is simply not updated.
+* The topic name created by the JDBC connector doesn't include the table's schema name. Using the topic.prefix connector configuration would mean that we'll have one connector per schema. At Bolt, we have a large number of schemas, which means we would need to create a large number of JDBC Source Connectors.
+* At Bolt, our downstream applications are already set up to use Debezium's data formats and topic naming conventions, we'd need to change our downstream application's decoding logic to the new data formats.
+* Row deletes are not captured.
+#### Option 3: Using VStream gRPC
+VTGate exposes a gRPC service called VStream. It is a server-side streaming service. Any gRPC client can subscribe to the [VStream](https://vitess.io/docs/concepts/vstream/) service to get a continuous stream of change events from the underlying MySQL instances. The change events that VStream emits have similar information to the MySQL binary logs of the underlying MySQL instances. A single VStream can even subscribe to multiple shards for a given keyspace, making it quite a convenient API to build CDC tools.
+
+Behind the scene, as shown in Figure 2, VStream reads change events from multiple VTTablets - one [VTTablet](https://vitess.io/docs/reference/programs/vttablet/) per shard. Therefore, it doesn't send duplicates from multiple VTTablets for a given shard. Each VTTablet is a proxy to its MySQL instance. A typical topology would include one master VTTablet and its corresponding MySQL instance, and multiple replica VTTablets, each of which is the proxy of its own replica MySQL instance. A VTTablet gets change events from its underlying MySQL instance and sends the change events back to VTGate, which in turn sends the change events back to VStream's gRPC client.
+
+When subscribing to the VStream service, the client can specify a VGTID and [Tablet Type](https://vitess.io/docs/concepts/tablet/#tablet-types) (e.g. MASTER, REPLICA). The VGTID tells the position from which VStream starts to send change events. Essentially, VGTID includes a list of (keyspace, shard, shard GTID) tuples. The Tablet Type tells which MySQL instance (primary or replica) in each shard do we read change events from.
+
+
+
+Figure 2. VStream architecture. Reference: https://vitess.io/docs/concepts/vstream
+
+Some advantages of using VStream gRPC are:
+* It is a simple way to receive change events from Vitess. It is also recommended in Vitess's documentation to use VStream to build CDC processes downstream.
+* VTGate hides the complexity of connecting to various source MySQL instances.
+* It has low latency since change events are streamed to the client as soon as they happen.
+* The change events include not only inserts and updates, but also deletes.
+* Probably one of the biggest advantages is that the change events contain the schema of each table. So you don't have to worry about fetching each table's schema in advance (by, for example, parsing DDLs or querying the table's definition).
+* The change events have VGTID included, which the CDC process can store and use as the offset from where to restart the CDC process next time.
+* Also importantly, VStream is designed to work well with Vitess operations such as Resharding and Moving Tables.
+
+There are also some disadvantages:
+* Although it includes table schemas, some important information is still missing. For example, the Enum and Set column types don't provide all the allowed values yet. This should be fixed in the next major release (Vitess 9) though.
+* Since VStream is a gRPC service, we cannot use the Debezium MySQL Connector out-of-the-box. However, it is quite straightforward to implement the gRPC client in other languages.
+
+All things considered, we've decided to use VStream gRPC to capture change events from Vitess and implement our Vitess Connector based on all the best practices of Debezium.
+Vitess Connector Deep Dive and Open Source
+After we've decided to implement our Vitess Connector, we started looking into the implementation details of various Debezium source connectors (MySQL, Postgres, SQLServer), to borrow some ideas. Almost all of them are implemented using a common Connector development framework. So it was clear we should develop the Vitess connector on top of it. We are very active users of the MySql Connector and we benefit from it being open-sourced, as it allows us to contribute to it things we were missing ourselves. So we decided we want to give back to the community and open-source the Vitess source connector code-base under the Debezium umbrella. Please feel free to learn more at Debezium Connector Vitess. We welcome and value any contributions.
+
+At a high level, as you can see below, connector instances are created in Kafka Connect workers. At the time of writing, you have two options to configure the connector to read from Vitess:
+
+#### Option 1 (recommended):
+As shown in Figure 3, each connector captures change events from all shards in a specific keyspace. If the keyspace is not sharded, the connector can still capture change events from the only shard in the keyspace. When it's the first time that the connector starts, it reads from the current VGTID position of all shards in the keyspace. Because it subscribes to all shards, it continuously captures change events from all shards and sends them to Kafka. It automatically supports the Vitess Reshard operation, there is no data loss, nor duplication.
+
+
+
+Figure 3. Each connector subscribes to all shards of a specific keyspace
+
+#### Option 2:
+As shown in Figure 4, each connector instance captures change events from a specific keyspace/shard pair. The connector instance gets the initial (the current) VGTID position of the keyspace/shard pair from VTCtld gRPC, which is another Vitess component. Each connector instance, independently, uses the VGTID it gets to subscribe to VStream gRPC and continuously capture change events from VStream and sends them to Kafka. To support the Vitess Reshard operation, you would need more manual operations.
+
+
+
+Figure 4. Each connector subscribes to one shard of a specific keyspace
+
+Internally, each connector task uses a gRPC thread to constantly receive change events from VStream and puts the events into an internal blocking queue. The connector task thread polls events out of the queue and sends them to Kafka, as can be seen in Figure 5.
+
+
+
+Figure 5. How each connector task works internally
+
+### Replication Challenges
+While we were implementing the Vitess Connector and digging deeper into Vitess, we've also realized a few challenges.
+Vitess Reshard
+The Vitess connector supports the Vitess Reshard operation when the connector is configured to subscribe to all shards of a given keyspace. VStream sends a VGTID that contains the shard GTID for all shards. Vitess Resharding is transparent to users. Once it's completed, Vitess will send the VGTID of the new shards. Therefore, the connector will use the new VGTID after reshard. However, you need to make sure that the connector is up and running when the reshard operation takes place. Especially please check that the offset topic of the connector has the new VGTID before deleting the old shards. This is because in case the old shards are deleted, VStream will not be able to recognize the VGTID from the old shards.
+
+If you decide to subscribe to one shard per connector, the connector does not provide out-of-the-box support for Vitess resharding. One manual workaround to support resharding is creating one new connector per target shard. For example, one new connector for the commerce/-80 shard, and another new connector for the commerce/80- shard. Bear in mind that because they're new connectors, by default, new topics will be created, however, you could use the Debezium logical topic router to route the records to the same kafka topics.
+#### Offset Management
+VStream includes a VGTID event in its response. We save the VGTID as the offset in the Kafka offset topic, so when the connector restarts, we can start from the saved VGTID. However, in rare cases when a transaction includes a huge amount of rows, VStream batches the change events into multiple responses, and only the last response has the VGTID. In such cases, we don't have the VGTID for every change event we receive. We have a few options to solve this particular issue:
+
+* We can buffer all the change events in memory and wait for the last response that contains the VGTID to arrive. So all events will have the correct VGTID associated with them. A few disadvantages are that we'll have higher latency before events are sent to Kafka. Also, memory usage could potentially increase quite a lot due to buffering. Buffering also adds complexity to the logic. We also have no control over the number of events VStream sends to us.
+* We can use the latest VGTID we have, which is the VGTID from the previous VStream response. If the connector fails and restarts when processing such a big transaction, it'll restart from the VGTID of the previous VStream response, thus reprocessing some events. Therefore, it has at-least-once event delivery semantics and it expects the downstream to be idempotent. Since most transactions are not big enough, most VStream responses will have VGTID in the response, so the chance of having duplicates is low. In the end, we chose this approach for its at-least-once delivery guarantee and its design simplicity.
+#### Schema Management
+VStream's response also includes a FIELD event. It's a special event that contains the schemas of the tables of which the rows are affected. For example, let's assume we have 2 tables, A and B. If we insert a few rows into table A, the FIELD event will only contain table A's schema. The VStream is smart enough to only include the FIELD event whenever necessary. For example, when a VStream client reconnects, or when a table's schema is changed.
+
+The older version of VStream includes only the column type (e.p. Integer, Varchar), no additional information such as whether the column is the primary key, whether the column has a default value, Decimal type's scale and precision, Enum type's allowed values, etc.
+
+The newer version (Vitess 8) of VStream starts to include more information on each column. This will help the connector to deserialize more accurately certain types and have a more precise schema in the change events sent to Kafka.
+### Future Development Work
+* We can use VStream's API to start streaming from the latest VGTID position, instead of getting the initial VGTID position from VTCtld gRPC. Doing so would eliminate the dependency from VTCtld.
+* We don't support automatically extracting the primary keys from the change events yet. Currently, by default, all change events sent to Kafka have null as the key, unless the message.key.columns connector configuration is specified. Vitess recently added flags of each column in the VStream FIELD event, which allows us to implement this feature soon.
+* Add support for initial snapshots to capture all existing data before streaming changes.
+Summary
+MySQL has been used to power most of our backend services at Bolt. Due to the considerable growth of the volume of data and operational complexity, Bolt started to evaluate Vitess for its scalability and its built-in features such as resharding.
+
+To capture data changes from Vitess, as what we've been doing with Debezium MySQL Connector, we've considered a few options. In the end, we have implemented our own Vitess Connector based on the common Debezium connector framework. While implementing the Vitess connector, we've encountered a few challenges. For example, support for the Vitess reshard operation, offset management, and schema management. We reasoned about ways to address the challenges and what we worked out as solutions.
+
+We've also received quite some interest from multiple communities in this project and we've decided to open-source [Vitess Connector](https://github.com/debezium/debezium-connector-vitess/) under the Debezium umbrella. Please feel free to learn more, and we welcome and value any contributions.
+
diff --git a/content/en/blog/2020-11-09-vitess-operator-for-kubernetes.md b/content/en/blog/2020-11-09-vitess-operator-for-kubernetes.md
new file mode 100644
index 000000000..5bf6f318e
--- /dev/null
+++ b/content/en/blog/2020-11-09-vitess-operator-for-kubernetes.md
@@ -0,0 +1,480 @@
+---
+author: 'Alkin Tezuysal'
+date: 2020-11-09
+slug: '2020-11-09-vitess-operator-for-kubernetes'
+tags: ['Vitess','MySQL','kubernetes','operator','cloud','GKE','sharding']
+title: 'Vitess Operator for Kubernetes'
+
+---
+### Introduction
+In this blog, I would like to explore [Vitess Operator for Kubernetes](https://github.com/planetscale/vitess-operator). This post demonstrates the sample implementation of [Vitess](https://vitess.io/) in Kubernetes topology. I also explore common DBA tasks by demonstrating how they are handled in the Vitess ecosystem. Vitess, out of the box, comes with a lot of tools and utilities that one has to either incorporate or develop to manage MySQL topology. Let’s take a look at the capabilities of Vitess in these areas and demonstrate how they are performed under the operator realm.
+
+#### Prerequisites
+* GKE Account
+* Kubectl
+* MySQL Client
+* Install vtctlclient locally
+ * go get vitess.io/vitess/go/cmd/vtctlclient
+* Download the Operator example files
+ * git clone git@github.com:vitessio/vitess.git
+* cd vitess/examples/operator
+* Run sample database implementation (Optional).
+
+#### Install the Vitess Operator
+
+$ gcloud container clusters create vitess-k8s-operator --cluster-version 1.14 --zone us-east1-b --enable-autoscaling --min-nodes 8 --max-nodes 12
+Creating cluster vitess-k8s-operator in us-east1-b... Cluster is being health-checked (master is healthy)...done.
+Created [https://container.googleapis.com/v1/projects/planetscale-dev/zones/us-east1-b/clusters/vitess-k8s-operator].
+To inspect the contents of your cluster, go to:
+https://console.cloud.google.com/kubernetes/workload_/gcloud/us-east1-b/vitess-k8s-operator?project=planetscale-dev
+kubeconfig entry generated for vitess-k8s-operator.
+
+NAME LOCATION MASTER_VERSION MASTER_IP MACHINE_TYPE NODE_VERSION NUM_NODES STATUS
+vitess-k8s-operator us-east1-b 1.14.10-gke.50 35.237.26.125 n1-standard-1 1.14.10-gke.50 3 RUNNING
+
+```
+$ cd vitess/examples/operator
+$ kubectl apply -f operator.yaml
+customresourcedefinition.apiextensions.k8s.io/etcdlockservers.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitessbackups.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitessbackupstorages.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitesscells.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitessclusters.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitesskeyspaces.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitessshards.planetscale.com created
+serviceaccount/vitess-operator created
+role.rbac.authorization.k8s.io/vitess-operator created
+rolebinding.rbac.authorization.k8s.io/vitess-operator created
+priorityclass.scheduling.k8s.io/vitess created
+priorityclass.scheduling.k8s.io/vitess-operator-control-plane created
+deployment.apps/vitess-operator created
+
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+vitess-operator-8454d86687-hv9lg 1/1 Running 0 41s
+```
+Bring up an initial test cluster
+```
+$ kubectl apply -f 101_initial_cluster.yaml
+vitesscluster.planetscale.com/example created
+secret/example-cluster-config created
+
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+example-etcd-faf13de3-1 1/1 Running 0 54s
+example-etcd-faf13de3-2 1/1 Running 0 54s
+example-etcd-faf13de3-3 1/1 Running 0 53s
+example-vttablet-zone1-2469782763-bfadd780 3/3 Running 1 54s
+example-vttablet-zone1-2548885007-46a852d0 2/3 Running 1 54s
+example-zone1-vtctld-1d4dcad0-59d8498459-54p68 1/1 Running 1 54s
+example-zone1-vtgate-bc6cde92-6fcfbb6666-5bcjc 1/1 Running 1 53s
+vitess-operator-8454d86687-hv9lg 1/1 Running 0 2m28s
+```
+Setup Port-forward
+```
+$ ./pf.sh &
+[1] 34424
+askdba:operator askdba$ You may point your browser to http://localhost:15000, use the following aliases as shortcuts:
+alias vtctlclient="vtctlclient -server=localhost:15999 -logtostderr"
+alias mysql="mysql -h 127.0.0.1 -P 15306 -u user"
+Hit Ctrl-C to stop the port forwards
+Forwarding from 127.0.0.1:15306 -> 3306
+Forwarding from [::1]:15306 -> 3306
+Forwarding from 127.0.0.1:15000 -> 15000
+Forwarding from [::1]:15000 -> 15000
+Forwarding from 127.0.0.1:15999 -> 15999
+Forwarding from [::1]:15999 -> 15999
+```
+```
+$ vtctlclient ApplySchema -sql="$(cat create_commerce_schema.sql)" commerce
+$ vtctlclient ApplyVSchema -vschema="$(cat vschema_commerce_initial.json)" commerce
+Handling connection for 15999
+New VSchema object:
+{
+ "tables": {
+ "corder": {
+ },
+ "customer": {
+ },
+ "product": {
+ }
+ }
+}
+If this is not what you expected, check the input data (as JSON parsing will skip unexpected fields).
+```
+
+After this section it’s recommended to to continue with the sample [database](https://vitess.io/docs/get-started/local/) steps to have a running database in this cluster(optional).
+### Database Operations
+
+In this section as we’ve mentioned above, I’d like to show how to perform functional operations as follows:
+
+1. Backup & Recovery
+2. Failover to Replica
+3. Schema Change
+
+#### Backup Vitess Tablet
+
+At this stage, our cluster within Kubernetes is up, but backups won’t work until we configure a place to store them.
+First, we will check existing backups and as you can see there are none.
+```$ kubectl get vitessbackups
+No resources found in default namespace.
+```
+In order to perform backups, we need to set up a backup storage bucket in GCS.
+Fill in your own values for all the my-* names. The bucket name in particular must be globally unique across all GCS users.
+Select GCP project
+```
+$ gcloud config set project [project-name]
+Updated property [core/project].
+```
+Create a GCS bucket
+```
+$ gsutil mb -l us-central1 -b on gs://my-vitess-operator-backup-bucket
+Creating gs://my-vitess-operator-backup-bucket/...
+```
+Create a GCP service account
+```
+$ gcloud iam service-accounts create my-backup-service-account
+Created service account [my-backup-service-account].
+```
+Grant the service account access to the bucket gsutil iam ch
+```
+$ gsutil iam ch serviceAccount:my-backup-service-account@planetscale-dev.iam.gserviceaccount.com:objectViewer,objectCreator,objectAdmin \
+gs://my-vitess-operator-backup-bucket
+```
+Create and download a key for the service account
+```
+$ gcloud iam service-accounts keys create ~/gcs_key.json --iam-account my-backup-service-account@planetscale-dev.iam.gserviceaccount.com
+created key [ccd65b5a198298f9ca07ee6ab901a2492ea142c7] of type [json] as [/Users/askdba/gcs_key.json]
+for [my-backup-service-account@planetscale-dev.iam.gserviceaccount.com]
+```
+Upload the service account key as a k8s Secret
+```
+$ kubectl create secret generic gcs-auth --from-file=gcs_key.json="$HOME/gcs_key.json"
+secret/gcs-auth created
+```
+Delete the local copy
+```
+$ rm ~/gcs_key.json
+```
+Check Tablet(s) status
+```
+$ kubectl describe vt | grep 'Tablets'
+ Desired Tablets: 2
+ Ready Tablets: 2
+ Tablets: 2
+ Updated Tablets: 2
+ Desired Tablets: 6
+ Ready Tablets: 6
+ Tablets: 6
+ Updated Tablets: 6
+```
+Note: The numbers will be different if you ran the optional sample database steps.
+At this stage, our cluster still has no association with the GSC bucket. We need to apply the correct backup method (xtrabackup) and bucket definition to the cluster via YAML file.
+
+Add a backup section under spec in each VitessCluster YAML before applying it. Since this is an idempotent operation we can edit and re-apply 101_initial_cluster.yaml file.
+```
+yaml
+spec:
+ backup:
+ engine: xtrabackup
+ locations:
+ - gcs:
+ bucket: my-vitess-backup-bucket
+ authSecret:
+ name: gcs-auth
+ key: gcs_key.json
+```
+```
+$ kubectl apply -f 101_initial_cluster.yaml
+vitesscluster.planetscale.com/example configured
+secret/example-cluster-config configured
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+example-90089e05-vitessbackupstorage-subcontroller 1/1 Running 0 29s
+example-commerce-x-x-vtbackup-init-c6db73c9 0/1 CrashLoopBackOff 1 29s
+example-etcd-faf13de3-1 1/1 Running 0 7m51s
+example-etcd-faf13de3-2 1/1 Running 0 7m51s
+example-etcd-faf13de3-3 1/1 Running 0 7m51s
+example-vttablet-zone1-2469782763-bfadd780 3/3 Running 0 15s
+example-vttablet-zone1-2548885007-46a852d0 3/3 Running 2 7m51s
+example-zone1-vtctld-1d4dcad0-59d8498459-qg4rv 1/1 Running 3 7m51s
+example-zone1-vtgate-bc6cde92-6fcfbb6666-gz6mm 1/1 Running 3 7m50s
+vitess-operator-8454d86687-m225m 1/1 Running 0 8m5s
+```
+
+If we had configured backups in the VitessCluster before the initial deployment, the operator would initialize backup storage automatically. Since we initially started the cluster without backups configured, the automatic backup initialization will fail:
+```
+$ kubectl logs example-commerce-x-x-vtbackup-init-c6db73c9
+ERROR: logging before flag.Parse: E0928 12:08:20.897936 1 syslogger.go:122] can't connect to syslog
+E0928 12:08:21.075860 1 vtbackup.go:177] Can't take backup: refusing to upload initial backup of empty database: the shard commerce/-
+already has at least one tablet that may be serving (zone1-2469782763); you must take a backup from a live tablet instead
+```
+This means we need to take a backup manually to initialize backup storage:
+```
+$ vtctlclient -logtostderr BackupShard commerce/-
+…
+I0928 15:13:29.818114 80158 main.go:64] I0928 12:13:29.492382 backup.go:162] I0928 12:13:29.492048 xtrabackupengine.go:309] xtrabackup stderr: 200928 12:13:29 [00] Streaming
+I0928 15:13:29.818123 80158 main.go:64] I0928 12:13:29.492404 backup.go:162] I0928 12:13:29.492133 xtrabackupengine.go:309] xtrabackup stderr: 200928 12:13:29 [00] ...done
+I0928 15:13:29.818131 80158 main.go:64] I0928 12:13:29.492415 backup.go:162] I0928 12:13:29.492217 xtrabackupengine.go:309] xtrabackup stderr: xtrabackup: Transaction log of lsn (2747504) to (2747513) was copied.
+I0928 15:13:29.818145 80158 main.go:64] I0928 12:13:29.594417 backup.go:162] I0928 12:13:29.593983 xtrabackupengine.go:309] xtrabackup stderr: 200928 12:13:29 completed OK!
+I0928 15:13:29.818160 80158 main.go:64] I0928 12:13:29.622037 backup.go:162] I0928 12:13:29.615593 xtrabackupengine.go:631] Found position: 3d370970-0182-11eb-9c3b-767b9aee7c34:1-9,3d3e599f-0182-11eb-88a6-a69501e63cc3:1-3
+I0928 15:13:29.818168 80158 main.go:64] I0928 12:13:29.622109 backup.go:162] I0928 12:13:29.615717 xtrabackupengine.go:115] Closing backup file backup.xbstream.gz-000
+I0928 15:13:30.160490 80158 main.go:64] I0928 12:13:29.999008 backup.go:162] I0928 12:13:29.996710 xtrabackupengine.go:115] Closing backup file backup.xbstream.gz-001
+I0928 15:13:30.321497 80158 main.go:64] I0928 12:13:30.160952 backup.go:162] I0928 12:13:30.160183 xtrabackupengine.go:115] Closing backup file backup.xbstream.gz-002
+I0928 15:13:30.493284 80158 main.go:64] I0928 12:13:30.333110 backup.go:162] I0928 12:13:30.332349 xtrabackupengine.go:115] Closing backup file backup.xbstream.gz-003
+I0928 15:13:30.594650 80158 main.go:64] I0928 12:13:30.433818 backup.go:162] I0928 12:13:30.433292 xtrabackupengine.go:115] Closing backup file backup.xbstream.gz-004
+I0928 15:13:30.752338 80158 main.go:64] I0928 12:13:30.591845 backup.go:162] I0928 12:13:30.590837 xtrabackupengine.go:115] Closing backup file backup.xbstream.gz-005
+I0928 15:13:30.862331 80158 main.go:64] I0928 12:13:30.702971 backup.go:162] I0928 12:13:30.702169 xtrabackupengine.go:115] Closing backup file backup.xbstream.gz-006
+I0928 15:13:31.027650 80158 main.go:64] I0928 12:13:30.868250 backup.go:162] I0928 12:13:30.867350 xtrabackupengine.go:115] Closing backup file backup.xbstream.gz-007
+I0928 15:13:31.209019 80158 main.go:64] I0928 12:13:31.049203 backup.go:162] I0928 12:13:31.047831 xtrabackupengine.go:162] Writing backup MANIFEST
+I0928 15:13:31.209046 80158 main.go:64] I0928 12:13:31.049501 backup.go:162] I0928 12:13:31.048074 xtrabackupengine.go:196] Backup completed
+I0928 15:13:31.209057 80158 main.go:64] I0928 12:13:31.049599 backup.go:162] I0928 12:13:31.048087 xtrabackupengine.go:115] Closing backup file MANIFEST
+```
+Within a minute or so, we should see the new backup appear when listing VitessBackup CRDs:
+```
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+example-90089e05-vitessbackupstorage-subcontroller 1/1 Running 0 6m41s
+example-etcd-faf13de3-1 1/1 Running 0 14m
+example-etcd-faf13de3-2 1/1 Running 0 14m
+example-etcd-faf13de3-3 1/1 Running 0 14m
+example-vttablet-zone1-2469782763-bfadd780 3/3 Running 0 6m27s
+example-vttablet-zone1-2548885007-46a852d0 3/3 Running 0 5m33s
+example-zone1-vtctld-1d4dcad0-59d8498459-qg4rv 1/1 Running 3 14m
+example-zone1-vtgate-bc6cde92-6fcfbb6666-gz6mm 1/1 Running 3 14m
+vitess-operator-8454d86687-m225m 1/1 Running 0 14m
+$ kubectl get vitessbackups
+NAME AGE
+example-commerce-x-x-20200928-121327-97ece60f-b8bc0ec7 2m18s
+```
+
+#### Restore Vitess Tablet
+Under normal circumstances, you shouldn’t ever have to restore a backup manually. Extending the number of tablets in a cluster will make the new vttablets automatically restore from the latest backup, then point themselves at the current master. Once caught up, they’ll go into serving state. Here’s how to do it manually for demonstration purposes.
+```
+$ vtctlclient ListAllTablets
+Handling connection for 15999
+zone1-2469782763 commerce - master 10.100.1.37:15000 10.100.1.37:3306 []
+zone1-2548885007 commerce - replica 10.100.4.17:15000 10.100.4.17:3306 []
+
+$ vtctlclient -logtostderr RestoreFromBackup zone1-2548885007
+I0928 15:28:04.783441 80378 main.go:64] I0928 12:28:04.172914 backup.go:247] I0928 12:28:04.172087 xtrabackupengine.go:398]
+Restore: returning replication position 3d370970-0182-11eb-9c3b-767b9aee7c34:1-9,3d3e599f-0182-11eb-88a6-a69501e63cc3:1-3
+I0928 15:28:04.783449 80378 main.go:64] I0928 12:28:04.173170 backup.go:247] I0928 12:28:04.172126 backup.go:308] Restore: starting mysqld for mysql_upgrade
+I0928 15:28:07.271912 80378 main.go:64] I0928 12:28:07.177696 backup.go:247] I0928 12:28:07.176828 backup.go:315] Restore: running mysql_upgrade
+I0928 15:28:08.752154 80378 main.go:64] I0928 12:28:08.657377 backup.go:247] I0928 12:28:08.656674 backup.go:326] Restore: populating local_metadata
+I0928 15:28:08.777876 80378 main.go:64] I0928 12:28:08.683299 backup.go:247] I0928 12:28:08.682656 backup.go:334] Restore: restarting mysqld after mysql_upgrade
+```
+List cells/keyspaces/shards/tablets
+```
+$ kubectl describe vtc
+Name: example-zone1-5abb61ae
+Namespace: default
+Labels: planetscale.com/cell=zone1
+ planetscale.com/cluster=example
+Annotations:
+API Version: planetscale.com/v2
+Kind: VitessCell
+Metadata:
+ Creation Timestamp: 2020-09-28T12:00:26Z
+ Generation: 1
+ Owner References:
+ API Version: planetscale.com/v2
+ Block Owner Deletion: true
+ Controller: true
+ Kind: VitessCluster
+ Name: example
+ UID: 323a1cfa-0182-11eb-9bbc-42010a8e01fe
+ Resource Version: 2105764
+$ kubectl get vtk
+NAME AGE
+example-commerce-fb9b866a 30m
+```
+List tablets for a particular keyspace/shard
+```
+$ kubectl get pods -l planetscale.com/component=vttablet,planetscale.com/keyspace=commerce,planetscale.com/shard=x-x
+NAME READY STATUS RESTARTS AGE
+example-vttablet-zone1-2469782763-bfadd780 3/3 Running 0 24m
+example-vttablet-zone1-2548885007-46a852d0 3/3 Running 0 23m
+```
+
+#### Failover Scenario
+In this section, let’s say we think our primary is not in good shape and we’d like to force a failover to the replica. We could use the drain feature of the operator to request a graceful failover to a replica. The operator will choose another suitable replica if one is available, healthy, and not itself drained.
+
+First check who is the current master of a given shard.
+```
+$ kubectl describe vts example-commerce-x-x-0f5afee6 | grep 'Master Alias'
+ Master Alias: zone1-2469782763
+```
+List tablets for that shard:
+```
+$ kubectl get pods -l planetscale.com/component=vttablet,planetscale.com/keyspace=commerce,planetscale.com/shard=x-x
+NAME READY STATUS RESTARTS AGE
+example-vttablet-zone1-2469782763-bfadd780 3/3 Running 0 28m
+example-vttablet-zone1-2548885007-46a852d0 3/3 Running 0 27m
+
+$ vtctlclient ListAllTablets
+Handling connection for 15999
+zone1-2469782763 commerce - master 10.100.1.37:15000 10.100.1.37:3306 []
+zone1-2548885007 commerce - replica 10.100.4.17:15000 10.100.4.17:3306 []
+```
+
+Drain mastership to some other tablet (e.g. to prepare to drain the Node it’s on) by annotating the current master tablet.
+```
+$ kubectl annotate pod example-vttablet-zone1-2469782763-bfadd780 drain.planetscale.com/started="Draining for blog"
+pod/example-vttablet-zone1-2469782763-bfadd780 annotated
+```
+
+Check failover status to see that a new master has been elected:
+
+```
+$ kubectl describe vts example-commerce-x-x-0f5afee6 | grep 'Master Alias'
+ Master Alias: zone1-2548885007
+```
+
+To avoid leaving any tablets in a drained state, we can undrain all of them by removing the annotation:
+```
+$ kubectl annotate pod -l planetscale.com/component=vttablet drain.planetscale.com/started-
+pod/example-vttablet-zone1-2469782763-bfadd780 annotated
+pod/example-vttablet-zone1-2548885007-46a852d0 annotated
+```
+
+At the time of writing this blog post there’s a huge amount of work by PlanetScale’s Vitess team to integrate failovers with the Orchestrator project. You can see a demo by Sugu here.
+### Schema Changes
+
+In this section, I’d like to demonstrate how to apply schema changes to the Vitess cluster. For this operation it’s possible to use the following methods:
+
+* ApplySchema - We’ve seen examples of ApplySchema during initialization steps of the cluster.
+* VTGate - This is a method to send direct DDL statements to the Vitess cluster.
+* Directly to MySQL using Online Schema Change tools such as pt-online-schema and gh-ost.
+* New and Experimental in Vitess 8.0: managed Online DDL. Users will use a special ALTER TABLE syntax, either via ApplySchema or via VTGate, and Vitess will automate an online schema change via gh-ost or pt-online-schema-change on relevant shards, giving the user visibility and control over the migration. The user will not need to know how to run gh-ost or online-schema-change. Vitess will take care of setup, discovery, throttling, and finally, cleanup.
+
+
+We find that there is no common practice to automating schema changes in production on large, sharded databases. Some companies develop their own automation flow. In many others that we’ve observed, schema changes are applied manually by database engineers. These engineers will have resolved which clusters are to be affected, which shards are in those clusters, which servers are in those shards, and what migration commands to run. The engineers will run gh-ost or online-schema-change manually, check for errors, and handle cleanup of tables and triggers etc. This is why we have created the Online DDL flow in [Vitess 8.0](https://github.com/vitessio/vitess/releases/tag/v8.0.0).
+
+This new functionality is still Experimental, and we will discuss it another time. Meanwhile, let’s describe how an engineer would manage online schema changes directly, by running pt-online-schema-change manually on relevant shards. Our friends at Percona were kind enough to build a Docker Image with Percona Toolkit and MySQL Client. Here’s where you can find this image that we’re going to use as a sidecar to our operator implementation. This will give us a clean cluster with a sidecar built in.
+
+Note: This will remove customized backup configurations in prior sections.
+```
+$ kubectl apply -f 101_initial_cluster.yaml.sidecar
+vitesscluster.planetscale.com/example configured
+
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+example-etcd-faf13de3-1 1/1 Running 0 119m
+example-etcd-faf13de3-2 1/1 Running 0 119m
+example-etcd-faf13de3-3 1/1 Running 0 119m
+example-vttablet-zone1-2469782763-bfadd780 4/4 Running 1 119m
+example-vttablet-zone1-2548885007-46a852d0 4/4 Running 1 119m
+example-zone1-vtctld-1d4dcad0-59d8498459-ksvqp 1/1 Running 2 119m
+example-zone1-vtgate-bc6cde92-6fcfbb6666-2j9cz 1/1 Running 2 119m
+vitess-operator-8454d86687-hj77s 1/1 Running 0 3d5h
+
+$ kubectl describe pods example-vttablet-zone1-2548885007-46a852d0 | grep percona
+ image: perconalab/percona-toolkit
+ name: percona-toolkit
+```
+Our default example consists of commerce database with three tables as seen below:
+```
+mysql> use commerce
+Database changed
+mysql> show tables;
++-----------------------+
+| Tables_in_vt_commerce |
++-----------------------+
+| corder |
+| customer |
+| product |
++-----------------------+
+3 rows in set (0.15 sec)
+```
+
+Let’s say we need to alter corder table and modify order_id column. We’ll assume this table is big so we would like to make this schema migration online with minimum impact to production workloads.
+
+```
+$ vtctlclient ListAllTablets
+Handling connection for 15999
+zone1-2469782763 commerce - master 10.100.4.34:15000 10.100.4.34:3306 []
+zone1-2548885007 commerce - replica 10.100.5.32:15000 10.100.5.32:3306 []
+
+$ kubectl exec --stdin --tty -c percona-toolkit example-vttablet-zone1-2469782763-bfadd780 -- bash
+```
+```
+bash-4.4$ pt-online-schema-change --socket /vt/socket/mysql.sock \
+--user=vt_dba h=localhost,D=vt_commerce,t=corder \
+--critical-load threads_running=10000000000 \
+--max-load threads_running:20 \
+--tries create_triggers:10000:1,drop_triggers:10000:1,swap_tables:10000:1,update_foreign_keys:10000:1,copy_rows:10000:1 \
+--set-vars lock_wait_timeout=5 \
+--pause-file=/tmp/pause-osc \
+--alter="MODIFY order_id bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT" \
+--recursion-method=none --execute
+No slaves found. See --recursion-method if host example-vttablet-zone1-2469782763-bfadd780 has slaves.
+Not checking slave lag because no slaves were found and --check-slave-lag was not specified.
+Operation, tries, wait:
+ analyze_table, 10, 1
+ copy_rows, 10000, 1
+ create_triggers, 10000, 1
+ drop_triggers, 10000, 1
+ swap_tables, 10000, 1
+ update_foreign_keys, 10000, 1
+Altering `vt_commerce`.`corder`...
+Creating new table...
+Created new table vt_commerce._corder_new OK.
+Altering new table...
+Altered `vt_commerce`.`_corder_new` OK.
+2020-10-12T12:09:05 Creating triggers...
+2020-10-12T12:09:05 Created triggers OK.
+2020-10-12T12:09:05 Copying approximately 1 rows...
+2020-10-12T12:09:05 Copied rows OK.
+2020-10-12T12:09:05 Analyzing new table...
+2020-10-12T12:09:05 Swapping tables...
+2020-10-12T12:09:05 Swapped original and new tables OK.
+2020-10-12T12:09:05 Dropping old table...
+2020-10-12T12:09:05 Dropped old table `vt_commerce`.`_corder_old` OK.
+2020-10-12T12:09:05 Dropping triggers...
+2020-10-12T12:09:05 Dropped triggers OK.
+Successfully altered `vt_commerce`.`corder`.
+
+$mysql -S /vt/socket/mysql.sock -u vt_dba
+mysql> show create table corder;
++--------+-----------------------------------------------------------------------------+
+| Table | Create Table |
++--------+-----------------------------------------------------------------------------+
+| corder | CREATE TABLE `corder` (
+ `order_id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+ `customer_id` bigint(20) DEFAULT NULL,
+ `sku` varbinary(128) DEFAULT NULL,
+ `price` bigint(20) DEFAULT NULL,
+ PRIMARY KEY (`order_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 |
++--------+-----------------------------------------------------------------------------+
+1 row in set (0.00 sec)
+```
+Let’s say we have followed the instructions and built a sharded cluster in such a state.
+```
+$ echo "show vitess_tablets;" | mysql --table
+Handling connection for 15306
++-------+----------+-------+------------+-------------+------------------+-------------+
+| Cell | Keyspace | Shard | TabletType | State | Alias | Hostname |
++-------+----------+-------+------------+-------------+------------------+-------------+
+| zone1 | commerce | - | MASTER | SERVING | zone1-2469782763 | 10.100.6.13 |
+| zone1 | commerce | - | REPLICA | SERVING | zone1-2548885007 | 10.100.5.37 |
+| zone1 | customer | - | MASTER | NOT_SERVING | zone1-1250593518 | 10.100.5.36 |
+| zone1 | customer | - | REPLICA | SERVING | zone1-3778123133 | 10.100.4.38 |
+| zone1 | customer | -80 | MASTER | SERVING | zone1-0120139806 | 10.100.1.59 |
+| zone1 | customer | -80 | REPLICA | SERVING | zone1-2289928654 | 10.100.2.25 |
+| zone1 | customer | 80- | MASTER | SERVING | zone1-0118374573 | 10.100.0.25 |
+| zone1 | customer | 80- | REPLICA | SERVING | zone1-4277914223 | 10.100.3.45 |
++-------+----------+-------+------------+-------------+------------------+-------------+
+```
+We would have to do the same for each shard’s MASTER.
+
+As you can see above the same approach can be followed with other tools such as gh-ost, proxysql etc. Sidecar approach is gaining popularity to tackle some of the operational database problems recently.
+
+### Conclusion
+
+In conclusion, Vitess Operator is a very powerful option when it comes to deploying your database cluster in Kubernetes. By testing possibilities and initial challenges I hope you found this demonstration useful. Kubernetes Operators are growing rapidly and empower the world’s best cloud services.
+
+#### Credits & References
+* Percona Engineering and Release team
+* PlanetScale Vitess Engineering team
diff --git a/content/en/blog/2020-11-30-how-to-deploy-django.md b/content/en/blog/2020-11-30-how-to-deploy-django.md
new file mode 100644
index 000000000..5c6adb9e5
--- /dev/null
+++ b/content/en/blog/2020-11-30-how-to-deploy-django.md
@@ -0,0 +1,274 @@
+---
+author: 'Alkin Tezuysal'
+date: 2020-11-30
+slug: '2020-11-30-how-to-deploy-django'
+tags: ['Vitess','MySQL','kubernetes','operator','cloud','GKE','sharding']
+title: 'Django with Vitess'
+
+---
+Django is a popular framework for Python application developers. It includes packages which make tasks like authorization and content administration easier. Django supports a number of [databases](https://docs.djangoproject.com/en/3.1/ref/databases/) including MySQL which makes it possible to run a Django application over Vitess without having to change the application code. Let’s take a look at how to combine the strengths of these two open source frameworks.
+
+We built this example using Vitess operator. You can see the details of the implementation in the blog post [*Vitess Operator for Kubernetes*](https://vitess.io/blog/2020-11-09-vitess-operator-for-kubernetes/).
+
+### Prerequisites
+
+* Python environment locally (3.X)
+* Kubernetes access (minikube, GKE)
+* Support for Django ORM via [Vitess](https://github.com/vitessio/vitess/blob/d234083743d1cc9757ef673bf89be1a4a299b0b0/support/django/README.md)
+
+For this example, we’re using GKE with an existing Kubernetes cluster. You can also do this via minikube locally.
+
+The Django example we are using is the ["weather app"](https://github.com/askdba/vitess_frameworks/tree/main/django/my_weather_app). We first launch the vitess operator using the provided [configuration](https://github.com/askdba/vitess_frameworks/blob/main/django/operator/operator.yaml).
+
+The following section includes these steps:
+* Creating a Vitess Operator pod
+* Building Vitess Cluster Components (1x primary tablet, 1x replica tablet, 3x etcd pods, 1x vtgate, 1x vtctld, 1x vitessbackup)
+* Create the ‘weatherapp’ database schema and users.
+
+```
+$ kubectl apply -f operator.yaml
+customresourcedefinition.apiextensions.k8s.io/etcdlockservers.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitessbackups.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitessbackupstorages.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitesscells.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitessclusters.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitesskeyspaces.planetscale.com created
+customresourcedefinition.apiextensions.k8s.io/vitessshards.planetscale.com created
+serviceaccount/vitess-operator created
+role.rbac.authorization.k8s.io/vitess-operator created
+rolebinding.rbac.authorization.k8s.io/vitess-operator created
+priorityclass.scheduling.k8s.io/vitess created
+priorityclass.scheduling.k8s.io/vitess-operator-control-plane created
+deployment.apps/vitess-operator created
+
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+vitess-operator-7f9c9d58f6-q5zlf 1/1 Running 0 20s
+```
+Initialize this cluster with a sample database called ‘weatherapp’ and user/password to access it will be embedded in the configuration [file](https://github.com/askdba/vitess_frameworks/blob/main/django/operator/101_initial_cluster.yaml.django). We are basically creating a database which is analogous to a keyspace in Vitess.
+
+```
+$ kubectl apply -f 101_initial_cluster.yaml.django
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+example-90089e05-vitessbackupstorage-subcontroller 1/1 Running 0 94s
+example-etcd-faf13de3-1 1/1 Running 0 94s
+example-etcd-faf13de3-2 1/1 Running 0 94s
+example-etcd-faf13de3-3 1/1 Running 0 94s
+example-vttablet-zone1-1542279354-edf1c7bf 2/3 Running 1 94s
+example-vttablet-zone1-3763665199-476cbd65 2/3 Running 2 94s
+example-weatherapp-x-x-vtbackup-init-75efaeeb 0/1 Completed 0 74s
+example-zone1-vtctld-1d4dcad0-67bfd56b8b-4dr9s 1/1 Running 2 94s
+example-zone1-vtgate-bc6cde92-59b88bc8d8-6wz86 1/1 Running 2 94s
+vitess-operator-7f9c9d58f6-q5zlf 1/1 Running 0 4m30s
+```
+As you can see this brings up a fully functional managed Vitess cluster with an unsharded keyspace consisting of one “Primary (Master)” and one “Replica”.
+
+
+### Step 1 - Set portforwards:
+
+```
+$ cat pf.sh ; ./pf.sh &
+#!/bin/sh
+
+kubectl port-forward --address localhost "$(kubectl get service --selector="planetscale.com/component=vtctld" -o name | head -n1)" 15000 15999 &
+process_id1=$!
+kubectl port-forward --address localhost "$(kubectl get service --selector="planetscale.com/component=vtgate,!planetscale.com/cell" -o name | head -n1)" 15306:3306 &
+process_id2=$!
+sleep 2
+echo "You may point your browser to http://localhost:15000, use the following aliases as shortcuts:"
+echo 'alias vtctlclient="vtctlclient -server=localhost:15999 -logtostderr"'
+echo 'alias mysql="mysql -h 127.0.0.1 -P 15306 -u user"'
+echo "Hit Ctrl-C to stop the port forwards"
+wait $process_id1
+wait $process_id2
+```
+Check Tablets:
+```
+$ vtctlclient ListAllTablets
+Handling connection for 15999
+zone1-1542279354 weatherapp - replica 10.100.1.75:15000 10.100.1.75:3306 []
+zone1-3763665199 weatherapp - master 10.100.3.57:15000 10.100.3.57:3306 [] 2020-10-16T09:06:59Z
+```
+### Step 2 - Verify database:
+
+```
+$ alias mysql="mysql -h 127.0.0.1 -P 15306 -u djangouser -p"
+$ mysql
+mysql: [Warning] Using a password on the command line interface can be insecure.
+Handling connection for 15306
+Welcome to the MySQL monitor. Commands end with ; or \g.
+Your MySQL connection id is 1
+Server version: 5.7.9-Vitess MySQL Community Server (GPL)
+
+Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
+
+Oracle is a registered trademark of Oracle Corporation and/or its
+affiliates. Other names may be trademarks of their respective
+owners.
+
+Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
+
+mysql> show databases;
++------------+
+| Databases |
++------------+
+| weatherapp |
++------------+
+1 row in set (0.16 sec)
+```
+
+### Step 3 - Setup application environment
+
+Now that we have set up the Vitess cluster with a MySQL backend, we can proceed to building the Django application. We will build a Django project using the django-admin command.
+
+
+```
+$ mkdir my_weather_app
+$ cd my_weather_app
+$ python3 -m venv env
+$ . env/bin/activate
+(env) askdba:my_weather_app askdba$
+$ pip install django
+$ django-admin startproject weatherapp
+$ cd weatherapp/
+$ ls -la
+total 8
+drwxr-xr-x 4 askdba staff 128 Oct 16 12:19 .
+drwxr-xr-x 4 askdba staff 128 Oct 16 12:18 ..
+-rwxr-xr-x 1 askdba staff 666 Oct 16 12:18 manage.py
+drwxr-xr-x 7 askdba staff 224 Oct 16 12:18 weatherapp
+Edit configuration file and update following section.
+$ vi weatherapp/settings.py [link to sample file]
+import os
+# Database
+# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
+
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'custom_db_backends.vitess',
+ 'OPTIONS': {
+ 'read_default_file': '/usr/local/mysql/my.cnf',
+ },
+ }
+}
+
+STATIC_ROOT = os.path.join(BASE_DIR, 'static')
+
+...
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ['127.0.0.1']
+
+# Application definition
+...
+```
+
+Copy customs_db_backends directory to your project directory. You can clone [Vitess](https://github.com/vitessio/vitess/tree/master) project to a local directory.
+
+```
+$ cp -r ~/vitess/support/django/custom_db_backends .
+$ vi /usr/local/mysql/my.cnf [link to sample my.cnf]
+[client]
+database = weatherapp
+user = djangouser
+password = ********
+port = 15306
+host = 127.0.0.1
+default-character-set = utf8mb4
+```
+
+### Step 4 - Install MySQL Client Connector
+
+```
+$ pip install mysqlclient
+Collecting mysqlclient
+ Using cached mysqlclient-2.0.1.tar.gz (87 kB)
+Using legacy 'setup.py install' for mysqlclient, since package 'wheel' is not installed.
+Installing collected packages: mysqlclient
+ Running setup.py install for mysqlclient ... done
+Successfully installed mysqlclient-2.0.1
+```
+
+### Step 5 - Build Django Framework over Vitess cluster
+
+At this stage, we’re ready to run the migrations to create initial Django metadata.
+
+```
+$ python manage.py migrate
+Operations to perform:
+ Apply all migrations: admin, auth, contenttypes, sessions
+Running migrations:
+ Applying contenttypes.0001_initial... OK
+ Applying auth.0001_initial... OK
+ Applying admin.0001_initial... OK
+ Applying admin.0002_logentry_remove_auto_add... OK
+ Applying admin.0003_logentry_add_action_flag_choices... OK
+ Applying contenttypes.0002_remove_content_type_name... OK
+ Applying auth.0002_alter_permission_name_max_length... OK
+ Applying auth.0003_alter_user_email_max_length... OK
+ Applying auth.0004_alter_user_username_opts... OK
+ Applying auth.0005_alter_user_last_login_null... OK
+ Applying auth.0006_require_contenttypes_0002... OK
+ Applying auth.0007_alter_validators_add_error_messages... OK
+ Applying auth.0008_alter_user_username_max_length... OK
+ Applying auth.0009_alter_user_last_name_max_length... OK
+ Applying auth.0010_alter_group_name_max_length... OK
+ Applying auth.0011_update_proxy_permissions... OK
+ Applying auth.0012_alter_user_first_name_max_length... OK
+ Applying sessions.0001_initial... OK
+```
+
+### Step 6 - Create an admin user
+
+Create an administrative user to access the Django Admin interface.
+```
+$ python manage.py createsuperuser
+Username (leave blank to use 'askdba'): askdba
+Email address: alkin@planetscale.com
+Password:
+Password (again):
+The password is too similar to the email address.
+This password is too short. It must contain at least 8 characters.
+Bypass password validation and create user anyway? [y/N]: y
+Superuser created successfully.
+(env) askdba:weatherapp askdba$
+```
+
+### Step 7 - Start Django daemon.
+
+```
+$ python manage.py runserver 127.0.0.1:8000
+Watching for file changes with StatReloader
+Performing system checks...
+
+System check identified no issues (0 silenced).
+October 16, 2020 - 09:37:02
+Django version 3.1.2, using settings 'weatherapp.settings'
+Starting development server at http://127.0.0.1:8000/
+Quit the server with CONTROL-C.
+```
+### Step 8 - Go to Django Admin page
+
+Point your browser to [http://127.0.0.1:8000/admin](http://127.0.0.1:8000/admin)
+
+
+
Admin Login Screen
+
+
User/Role Management Screen
+
+
+You may continue to build the application from this point by following this [example](https://www.digitalocean.com/community/tutorials/how-to-build-a-weather-app-in-django).
+
+
+### Conclusion
+
+Vitess is a very powerful sharding framework that comes with a built-in control plane that allows backend developers to adapt their applications easily. Combining powerful application frameworks such as Django allows developers to create scalable applications out of the box with the power of open source tooling.
+
+### References
+[How To Make a Django Blog App and Connect it to MySQL](https://www.digitalocean.com/community/tutorials/how-to-create-a-django-app-and-connect-it-to-a-database)
+
+[Getting Started With Django: Build A Weather App](https://www.digitalocean.com/community/tutorials/how-to-build-a-weather-app-in-django)
+
+[Django MySQL Notes](https://docs.djangoproject.com/en/3.1/ref/databases/#mysql-notes)
diff --git a/content/en/blog/2020-12-15-vitess-at-kubecon-na.md b/content/en/blog/2020-12-15-vitess-at-kubecon-na.md
new file mode 100644
index 000000000..b0652dbef
--- /dev/null
+++ b/content/en/blog/2020-12-15-vitess-at-kubecon-na.md
@@ -0,0 +1,53 @@
+---
+author: 'Deepthi Sigireddi'
+date: 2020-12-15
+slug: '2020-12-15-vitess-at-kubecon-na'
+tags: ['Vitess','MySQL','conference']
+title: 'Kubecon+CloudNativeCon NA 2020'
+description: 'Vitess presence at Kubecon CloudNativeCon North America 2020 (Virtual) '
+---
+The Vitess [team](https://twitter.com/vitessio) had a successful presence at Kubecon + CloudNativeCon North America 2020. This year's event was in a virtual format. It ran from Nov 17-20.
+We had 2 talks, a project booth and 3 office hours sessions. In addition, we had mentions in 2 major keynote talks.
+
+## Day 1
+Day 1 was a non-keynote day. It opened relatively quietly for us. We had 25+ attendees at our office hours session hosted by maintainers **[Alkin Tezuysal](https://twitter.com/ask_dba)**, **[Deepthi Sigireddi](https://twitter.com/ATechGirl)** and **[Derek Perkins](https://twitter.com/derek_perkins)**.
+Some people wandered into the project booth.
+
+
+
Vitess Project Booth
+
+
+## Day 2
+This was our big day. Not only did we have both Vitess talks on this day, but we also had mentions in keynotes.
+First, **Priyanka Sharma, CNCF GM** spoke about [how HubSpot uses Vitess and Kubernetes](https://www.youtube.com/watch?v=Uga3XnFG0to).
+
+
+
Vitess + Kubernetes @ HubSpot
+
+
+Then, **Constance Caramanolis, Conference Co-Chair**, featured Vitess in the [Project Updates Keynote](https://www.youtube.com/watch?v=uNpZA5fz5o8).
+
+
+
Vitess Project Update
+
+
+**Constance** also featured Slack's Vitess migration as an End-User Story.
+
+
+
Vitess @ Slack
+
+
+**[Guido Iaquinti](https://twitter.com/guidoiaquinti)** and **[Rafael Chacon](https://twitter.com/rafaelchacon)** of Slack spoke about how they [migrated their MySQL Clusters into Vitess](https://www.youtube.com/watch?v=k51L0xdClGQ).
+We wrapped up the day with the [Vitess Maintainer talk](https://www.youtube.com/watch?v=Nqzhgzg5WCw) which featured demos from a number of our maintainers.
+
+Our project office hours featuring **[Sugu Sougoumarane](https://twitter.com/ssougou)** and **Rafael** drew 120 attendees.
+
+## Day 3
+Another quiet day. We had our [monthly public meeting](https://zoom.us/rec/share/A8tOSvILYkUx65bizFvkph7_aFNOyWkEr3VxqqJl5XealI7aKSodVVdaJ5I-mArd.ftVtXObha-BCERxo) (Access Passcode: AKi00k&@) on this day, so we did not host office hours.
+
+## Day 4
+We hosted our last project office hours with **Alkin**, **Deepthi** and **[Paul Hemberger](https://github.com/pH14)**. 65 people attended the session.
+
+## Project Booth
+We had over 350 unique visitors to the project booth. We had maintainers available to answer questions but most visitors chose to view documents and not engage with the maintainers at the booth.
+Thank you to everyone who attended the talks or visited the booth. Special thanks to our maintainer team for making all of this possible.
diff --git a/content/en/docs/7.0/user-guides/region-sharding.md b/content/en/docs/7.0/user-guides/region-sharding.md
deleted file mode 100644
index 4176a8bf9..000000000
--- a/content/en/docs/7.0/user-guides/region-sharding.md
+++ /dev/null
@@ -1,294 +0,0 @@
----
-title: Region-based Sharding
-weight: 8
----
-
-{{< info >}}
-This guide follows on from the Get Started guides. Please make sure that you have a [local](../../get-started/local) installation ready. You should also have already gone through the [MoveTables](../move-tables) and [Resharding](../resharding) tutorials.
-{{< /info >}}
-
-## Preparation
-Having gone through the Resharding tutorial, you should be familiar with [VSchema](../../concepts/vschema) and [Vindexes](../../reference/vindexes).
-In this tutorial, we will create a sharded keyspace using a location-based vindex. We will create 4 shards (-40, 40-80, 80-c0, c0-).
-The location will be denoted by a `country` column.
-
-## Schema
-We will create 2 tables in this example.
-```text
-CREATE TABLE customer (
- id int NOT NULL,
- fullname varbinary(256),
- nationalid varbinary(256),
- country varbinary(256),
- primary key(id)
- );
-CREATE TABLE customer_lookup (
- id int NOT NULL,
- keyspace_id varbinary(256),
- primary key(id)
- );
-```
-
-The customer table is the main table we want to shard using country. The lookup table will help us do that.
-
-## Region Vindex
-We will use a `region_json` vindex to compute the keyspace_id for a customer row using the (id, country) fields.
-Here's what the vindex definition looks like:
-```text
- "region_vdx": {
- "type": "region_json",
- "params": {
- "region_map": "/vt/examples/region_sharding/countries.json",
- "region_bytes": "1"
- }
- },
-```
-And we use it thus:
-```text
- "customer": {
- "column_vindexes": [
- {
- "columns": ["id", "country"],
- "name": "region_vdx"
- },
-```
-This vindex uses a byte mapping of countries provided in a JSON file and combines that with the id column in the customer table to compute the keyspace_id. In this example, we are using 1 byte. You can use 1 or 2 bytes. With 2 bytes, 65536 distinct locations can be supported. The byte value of the country(or other location identifier) is prefixed to a hash value computed from the id to produce the keyspace_id.
-
-The lookup table is used to store the id to keyspace_id mapping. We connect it to the customer table as follows:
-We first define a lookup vindex:
-```text
- "customer_region_lookup": {
- "type": "consistent_lookup_unique",
- "params": {
- "table": "customer_lookup",
- "from": "id",
- "to": "keyspace_id"
- },
- "owner": "customer"
- },
-```
-Then we create it as a vindex on the customer table:
-```text
- "customer": {
- "column_vindexes": [
- {
- "columns": ["id", "country"],
- "name": "region_vdx"
- },
- {
- "column": "id",
- "name": "customer_region_lookup"
- }
- ]
- }
-```
-
-The lookup table could be unsharded or sharded. In this example, we have chosen to shard the lookup table also. If the goal of region-based sharding is data locality, it makes sense to co-locate the lookup data with the main customer data.
-We first define an `identity` vindex:
-```text
- "identity": {
- "type": "binary"
- }
-```
-Then we create it as a vindex on the lookup table:
-```text
- "customer_lookup": {
- "column_vindexes": [
- {
- "column": "keyspace_id",
- "name": "identity"
- }
- ]
- },
-```
-
-This is what the JSON file contains:
-```text
-{
- "United States": 1,
- "Canada": 2,
- "France": 64,
- "Germany": 65,
- "China": 128,
- "Japan": 129,
- "India": 192,
- "Indonesia": 193
-}
-```
-The values for the countries have been chosen such that 2 countries fall into each shard.
-
-## Start the Cluster
-
-Start by copying the region_sharding example included with Vitess to your preferred location.
-```sh
-cp -r /usr/local/vitess/examples/region_sharding ~/my-vitess/examples/region_sharding
-cd ~/my-vitess/examples/region_sharding
-```
-
-The VSchema for this tutorial uses a config file. You will need to edit the value of the `region_map` parameter in the vschema file `main_vschema.json`.
-For example:
-```text
-"region_map": "/home/user/my-vitess/examples/region_sharding/countries.json",
-```
-
-Now start the cluster
-```sh
-./101_initial_cluster.sh
-```
-
-You should see output similar to the following:
-
-```text
-~/my-vitess-example> ./101_initial_cluster.sh
-add /vitess/global
-add /vitess/zone1
-add zone1 CellInfo
-etcd start done...
-Starting vtctld...
-Starting MySQL for tablet zone1-0000000100...
-Starting vttablet for zone1-0000000100...
-HTTP/1.1 200 OK
-Date: Thu, 21 May 2020 01:05:26 GMT
-Content-Type: text/html; charset=utf-8
-
-Starting MySQL for tablet zone1-0000000200...
-Starting vttablet for zone1-0000000200...
-HTTP/1.1 200 OK
-Date: Thu, 21 May 2020 01:05:31 GMT
-Content-Type: text/html; charset=utf-8
-
-Starting MySQL for tablet zone1-0000000300...
-Starting vttablet for zone1-0000000300...
-HTTP/1.1 200 OK
-Date: Thu, 21 May 2020 01:05:35 GMT
-Content-Type: text/html; charset=utf-8
-
-Starting MySQL for tablet zone1-0000000400...
-Starting vttablet for zone1-0000000400...
-HTTP/1.1 200 OK
-Date: Thu, 21 May 2020 01:05:40 GMT
-Content-Type: text/html; charset=utf-8
-
-W0520 18:05:40.443933 6824 main.go:64] W0521 01:05:40.443180 reparent.go:185] master-elect tablet zone1-0000000100 is not the shard master, proceeding anyway as -force was used
-W0520 18:05:40.445230 6824 main.go:64] W0521 01:05:40.443744 reparent.go:191] master-elect tablet zone1-0000000100 is not a master in the shard, proceeding anyway as -force was used
-W0520 18:05:40.496253 6841 main.go:64] W0521 01:05:40.495599 reparent.go:185] master-elect tablet zone1-0000000200 is not the shard master, proceeding anyway as -force was used
-W0520 18:05:40.496508 6841 main.go:64] W0521 01:05:40.495647 reparent.go:191] master-elect tablet zone1-0000000200 is not a master in the shard, proceeding anyway as -force was used
-W0520 18:05:40.537548 6858 main.go:64] W0521 01:05:40.536985 reparent.go:185] master-elect tablet zone1-0000000300 is not the shard master, proceeding anyway as -force was used
-W0520 18:05:40.537758 6858 main.go:64] W0521 01:05:40.537041 reparent.go:191] master-elect tablet zone1-0000000300 is not a master in the shard, proceeding anyway as -force was used
-W0520 18:05:40.577854 6875 main.go:64] W0521 01:05:40.577407 reparent.go:185] master-elect tablet zone1-0000000400 is not the shard master, proceeding anyway as -force was used
-W0520 18:05:40.578042 6875 main.go:64] W0521 01:05:40.577448 reparent.go:191] master-elect tablet zone1-0000000400 is not a master in the shard, proceeding anyway as -force was used
-...
-Waiting for vtgate to be up...
-vtgate is up!
-Access vtgate at http://localhost:15001/debug/status
-
-```
-
-You can also verify that the processes have started with `pgrep`:
-
-```bash
-~/my-vitess-example> pgrep -fl vtdataroot
-3920 etcd
-4030 vtctld
-4173 mysqld_safe
-4779 mysqld
-4817 vttablet
-4901 mysqld_safe
-5426 mysqld
-5461 vttablet
-5542 mysqld_safe
-6100 mysqld
-6136 vttablet
-6231 mysqld_safe
-6756 mysqld
-6792 vttablet
-6929 vtgate
-```
-
-_The exact list of processes will vary. For example, you may not see `mysqld_safe` listed._
-
-If you encounter any errors, such as ports already in use, you can kill the processes and start over:
-
-```sh
-pkill -9 -e -f '(vtdataroot|VTDATAROOT)' # kill Vitess processes
-rm -rf vtdataroot
-```
-
-## Aliases
-
-For ease-of-use, Vitess provides aliases for `mysql` and `vtctlclient`. These are automatically created when you start the cluster.
-```bash
-source ./env.sh
-```
-
-Setting up aliases changes `mysql` to always connect to Vitess for your current session. To revert this, type `unalias mysql && unalias vtctlclient` or close your session.
-
-## Connect to your cluster
-
-You should now be able to connect to the VTGate server that was started in `101_initial_cluster.sh`:
-
-```bash
-~/my-vitess-example> mysql
-Welcome to the MySQL monitor. Commands end with ; or \g.
-Your MySQL connection id is 2
-Server version: 5.7.9-Vitess (Ubuntu)
-
-Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
-
-Oracle is a registered trademark of Oracle Corporation and/or its
-affiliates. Other names may be trademarks of their respective
-owners.
-
-Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
-
-mysql> show tables;
-+-------------------+
-| Tables_in_vt_main |
-+-------------------+
-| customer |
-| customer_lookup |
-+-------------------+
-2 rows in set (0.01 sec)
-```
-
-## Insert some data into the cluster
-```bash
-~/my-vitess-example> mysql < insert_customers.sql
-```
-
-## Examine the data we just inserted
-```text
-mysql> use main/-40;
-Database changed
-
-mysql> select * from customer;
-+----+-----------------+-------------+---------------+
-| id | fullname | nationalid | country |
-+----+-----------------+-------------+---------------+
-| 1 | Philip Roth | 123-456-789 | United States |
-| 2 | Gary Shteyngart | 234-567-891 | United States |
-| 3 | Margaret Atwood | 345-678-912 | Canada |
-| 4 | Alice Munro | 456-789-123 | Canada |
-+----+-----------------+-------------+---------------+
-4 rows in set (0.01 sec)
-
-mysql> select id,hex(keyspace_id) from customer_lookup;
-+----+--------------------+
-| id | hex(keyspace_id) |
-+----+--------------------+
-| 1 | 01166B40B44ABA4BD6 |
-| 2 | 0106E7EA22CE92708F |
-| 3 | 024EB190C9A2FA169C |
-| 4 | 02D2FD8867D50D2DFE |
-+----+--------------------+
-4 rows in set (0.00 sec)
-```
-You can see that only data from US and Canada exists in this shard.
-Repeat this for the other shards (40-80, 80-c0 and c0-) and see that each shard contains 4 rows in customer table and the 4 corresponding rows in the lookup table.
-
-You can now teardown your example:
-
-```bash
-./201_teardown.sh
-rm -rf vtdataroot
-```
diff --git a/content/en/docs/contributing/build-on-centos.md b/content/en/docs/contributing/build-on-centos.md
index 15d7e7141..06b571a1d 100644
--- a/content/en/docs/contributing/build-on-centos.md
+++ b/content/en/docs/contributing/build-on-centos.md
@@ -12,18 +12,27 @@ The following has been verified to work on __CentOS 7__. If you are new to Vites
## Install Dependencies
-### Install Go 1.13+
+### Install Go 1.15+
-[Download and install](http://golang.org/doc/install) Golang 1.13. For example, at writing:
+[Download and install](http://golang.org/doc/install) Golang 1.15. For example, at writing:
```
-curl -O https://dl.google.com/go/go1.13.9.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.13.9.linux-amd64.tar.gz
+curl -O https://golang.org/dl/go1.15.6.linux-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.15.6.linux-amd64.tar.gz
```
Make sure to add go to your bashrc:
```
+# Additions to ~/.bashrc file
+
+# Add go PATH
export PATH=$PATH:/usr/local/go/bin
+
+# Add GOROOT
+export GOROOT=/usr/local/go/
+
+# Add GOPATH
+export GOPATH=/home//go
```
### Packages from CentOS repos
@@ -69,8 +78,8 @@ Set environment variables that Vitess will require. It is recommended to put the
```
# Additions to ~/.bashrc file
-# Add go PATH
-export PATH=$PATH:/usr/local/go/bin
+#VTDATAROOT
+export VTDATAROOT=/tmp/vtdataroot
# Vitess binaries
export PATH=~/vitess/bin:${PATH}
diff --git a/content/en/docs/contributing/build-on-macos.md b/content/en/docs/contributing/build-on-macos.md
index 0ca5b0c4f..f168a7e8b 100644
--- a/content/en/docs/contributing/build-on-macos.md
+++ b/content/en/docs/contributing/build-on-macos.md
@@ -21,16 +21,18 @@ The following has been verified to work on __macOS Mojave__. If you are new to V
[Install Homebrew](http://brew.sh/). From here you should be able to install:
```shell
-brew install go@1.13 automake git curl wget mysql@5.7 etcd
+brew install go@1.15 automake git curl wget mysql@5.7
```
-Add `mysql@5.7` and `go@1.13` to your `PATH`:
+Add `mysql@5.7` and `go@1.15` to your `PATH`:
```shell
echo 'export PATH="/usr/local/opt/mysql@5.7/bin:$PATH"' >> ~/.bash_profile
-echo 'export PATH="/usr/local/opt/go@1.13/bin:$PATH"' >> ~/.bash_profile
+echo 'export PATH="/usr/local/opt/go@1.15/bin:$PATH"' >> ~/.bash_profile
```
+Do not install etcd via brew otherwise it will not be the version that is supported. Let it be installed when running make build.
+
Do not setup MySQL or etcd to restart at login.
## Build Vitess
@@ -63,7 +65,7 @@ The unit tests require that you first install a Java runtime. This is required f
```shell
brew tap adoptopenjdk/openjdk
brew cask install adoptopenjdk8
-brew cask info java
+brew info java
```
You will also need to install `ant` and `maven`:
@@ -91,3 +93,8 @@ Error: 105: Key already exists (/vitess/zone1) [6]
Error: 105: Key already exists (/vitess/global) [6]
```
+### /tmp/mysql.sock Already In Use
+This error occurs because mysql is serving on the same port that vttgate requires. To solve this issue stop mysql service. If you have installed mysql via brew as specified above you should run:
+```shell
+brew services stop mysql@5.7
+```
diff --git a/content/en/docs/contributing/build-on-ubuntu.md b/content/en/docs/contributing/build-on-ubuntu.md
index b0a944b28..a4c43afbf 100644
--- a/content/en/docs/contributing/build-on-ubuntu.md
+++ b/content/en/docs/contributing/build-on-ubuntu.md
@@ -12,18 +12,27 @@ The following has been verified to work on __Ubuntu 19.10__ and __Debian 10__. I
## Install Dependencies
-### Install Go 1.13+
+### Install Go 1.15+
-[Download and install](http://golang.org/doc/install) Golang 1.13. For example, at writing:
+[Download and install](http://golang.org/doc/install) Golang 1.15. For example, at writing:
```
-curl -O https://dl.google.com/go/go1.13.9.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.13.9.linux-amd64.tar.gz
+curl -O https://dl.google.com/go/go1.15.6.linux-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.15.6.linux-amd64.tar.gz
```
Make sure to add go to your bashrc:
```
+# Additions to ~/.bashrc file
+
+# Add go PATH
export PATH=$PATH:/usr/local/go/bin
+
+# Add GOROOT
+export GOROOT=/usr/local/go/
+
+# Add GOPATH
+export GOPATH=/home//go
```
### Packages from apt repos
@@ -82,8 +91,8 @@ Set environment variables that Vitess will require. It is recommended to put the
```
# Additions to ~/.bashrc file
-# Add go PATH
-export PATH=$PATH:/usr/local/go/bin
+#VTDATAROOT
+export VTDATAROOT=/tmp/vtdataroot
# Vitess binaries
export PATH=~/vitess/bin:${PATH}
diff --git a/content/en/docs/contributing/code-reviews.md b/content/en/docs/contributing/code-reviews.md
index c8c26aaaa..14953957b 100644
--- a/content/en/docs/contributing/code-reviews.md
+++ b/content/en/docs/contributing/code-reviews.md
@@ -19,10 +19,11 @@ In order to avoid disruption, the following concerns need to be kept in mind:
Every GitHub pull request must go through a code review and get approved before it will be merged into the master branch.
Every pull request should meet the following requirements:
+* Use the [Pull Request Template](https://github.com/vitessio/vitess/blob/master/.github/pull_request_template.md)
* Adhere to the [Go coding guidelines](https://golang.org/doc/effective_go.html) and watch out for these [common errors](https://github.com/golang/go/wiki/CodeReviewComments).
* Contain a description message that is as detailed as possible. Here is a great example https://github.com/vitessio/vitess/pull/6543.
* Pass all CI tests that run on PRs.
-* For bigger changes, it is a good idea to start by creating an issue - this is where you can discuss the feature and why it's important.
+* For bigger changes, it is a good idea to start by creating an RFC (Request for Commnent) issue - this is where you can discuss the feature and why it's important.
Once that is in place, you can create the PR, as a solution to the problem described in the issue. Separating the need and the solution this way makes discussions easier and more focused.
### Testing
diff --git a/content/en/docs/contributing/github-workflow.md b/content/en/docs/contributing/github-workflow.md
index baf0c917d..a094f242e 100644
--- a/content/en/docs/contributing/github-workflow.md
+++ b/content/en/docs/contributing/github-workflow.md
@@ -138,3 +138,9 @@ Once your pull request is merged:
* close the GitHub issue (if it wasn't automatically closed)
* delete your local topic branch (`git branch -d new-feature`)
+## Submitting Issues
+
+If you have a significant change to add, you need to [create an issue](https://github.com/vitessio/vitess/issues) prior to creating a Pull Request. This issue should be used to explain what you're planning to work on, to track progress, and design decisions.
+
+Or if you'd like to report a bug you've found within Vitess you can also create an issue.
+
diff --git a/content/en/docs/faq/_index.md b/content/en/docs/faq/_index.md
index a44c4d021..17c5fe00a 100644
--- a/content/en/docs/faq/_index.md
+++ b/content/en/docs/faq/_index.md
@@ -2,7 +2,46 @@
title: FAQ
description: Frequently Asked Questions about Vitess
weight: 8
-featured: true
+docs_nav_disable_expand: true
aliases: ['/docs/user-guides/faq/']
---
+## Where can I ask questions about Vitess?
+
+Our most popular channel and the one we recommend for asking questions you may have is our Slack located [here](https://vitess.io/slack).
+
+We have a number of other options that can be used as well listed [here](https://vitess.io/community/).
+
+Please do note that we request that you do not ask individual project members for support. Instead please use these channels where the whole community can help you and benefit from the solutions provided. Thanks!
+
+## What are the key slack channels to join?
+
+There are many channels available and we encourage you to join as many or as few as interest you. Some of the most popular channels are listed below:
+
+* #kubernetes
+* #monitoring
+* #operator
+* #orchestrator-integration
+* #releases
+* #vreplication
+
+## How can I contribute a Pull Request to Vitess?
+
+We always enjoy having new contributors to Vitess. Just be sure to read the information [here](https://vitess.io/docs/contributing/) to start.
+
+If you are already familiar with Vitess and you'd like information on how to file a Pull Request or submit an Issue request check out the following links:
+
+* [Pull Requests](https://vitess.io/docs/contributing/github-workflow/#sending-pull-requests)
+* [Issue](https://vitess.io/docs/contributing/github-workflow/#submitting-issues)
+
+## What are good videos to watch to get started learning about Vitess?
+
+We have a number of [recorded presentations and videos](https://vitess.io/docs/resources/presentations/) that can be watched to start learning about Vitess.
+
+PlanetScale also hosts a [number of videos](https://www.planetscale.com/resources) on Vitess.
+
+* For a curated list please check out a PlanetScale blog post [here](https://www.planetscale.com/blog/videos-intro-to-vitess-its-powerful-capabilities-and-how-to-get-started).
+
+## Where can I read additional FAQs?
+
+PlanetScale hosts a knowledge base for Vitess. This additional resource is available [here](https://planetscale.freshdesk.com/support/solutions).
\ No newline at end of file
diff --git a/content/en/docs/faq/configuration.md b/content/en/docs/faq/configuration.md
deleted file mode 100644
index f73ad8a8e..000000000
--- a/content/en/docs/faq/configuration.md
+++ /dev/null
@@ -1,40 +0,0 @@
----
-title: Configuration
-description: Frequently Asked Questions about Configuration
----
-
-## Does the application need to know about the sharding scheme underneath Vitess?
-
-The application does not need to know about how the data is sharded. This information is stored in a VSchema which the VTGate servers use to automatically route your queries. This allows the application to connect to Vitess and use it as if it’s a single giant database server.
-
-### Can I override the default db name from vt_xxx to my own?
-
-Yes. You can start vttablet with the `-init_db_name_override` command line option to specify a different db name. There is no downside to performing this override
-
-### How do I connect to vtgate using MySQL protocol?
-
-If you look at the example [vtgate-up.sh](https://github.com/vitessio/vitess/blob/master/examples/legacy_local/scripts/vtgate-up.sh) script, you'll see the following lines:
-
-```shell
--mysql_server_port $mysql_server_port \
--mysql_server_socket_path $mysql_server_socket_path \
--mysql_auth_server_static_file "./mysql_auth_server_static_creds.json" \
-```
-
-In this example, vtgate accepts MySQL connections on port 15306 and the authentication info is stored in the json file. So, you should be able to connect to it using the following command:
-
-```shell
-mysql -h 127.0.0.1 -P 15306 -u mysql_user --password=mysql_password
-```
-
-## I cannot start a cluster, and see these errors in the logs: Could not open required defaults file: /path/to/my.cnf
-
-Most likely this means that AppArmor is running on your server and is preventing Vitess processes from accessing the my.cnf file. The workaround is to uninstall AppArmor:
-
-```shell
-sudo service apparmor stop
-sudo service apparmor teardown
-sudo update-rc.d -f apparmor remove
-```
-
-You may also need to reboot the machine after this. Many programs automatically install AppArmor, so you may need to uninstall again.
diff --git a/content/en/docs/faq/queries.md b/content/en/docs/faq/queries.md
deleted file mode 100644
index 12c98b7d0..000000000
--- a/content/en/docs/faq/queries.md
+++ /dev/null
@@ -1,40 +0,0 @@
----
-title: Queries
-description: Frequently Asked Questions about Queries
----
-
-
-## Can I address a specific shard if I want to?
-
-If necessary, you can access a specific shard by connecting to it using the shard specific database name. For a keyspace ks and shard -80, you would connect to ks:-80.
-
-## How do I choose between master vs. replica for queries?
-
-You can qualify the keyspace name with the desired tablet type using the @ suffix. This can be specified as part of the connection as the database name, or can be changed on the fly through the USE command.
-
-For example, `ks@master` will select `ks` as the default keyspace with all queries being sent to the master. Consequently `ks@replica` will load balance requests across all `REPLICA` tablet types, and `ks@rdonly` will choose `RDONLY`.
-
-You can also specify the database name as `@master`, etc, which instructs Vitess that no default keyspace was specified, but that the requests are for the specified tablet type.
-
-If no tablet type was specified, then VTGate chooses its default, which can be overridden with the `-default_tablet_type` command line argument.
-
-## There seems to be a 10 000 row limit per query. What if I want to do a full table scan?
-
-Vitess supports different modes. In OLTP mode, the result size is typically limited to a preset number (10 000 rows by default). This limit can be adjusted based on your needs.
-
-However, OLAP mode has no limit to the number of rows returned. In order to change to this mode, you may issue the following command before executing your query:
-
-```shell
-set workload='olap'
-```
-You can also set the workload to `dba mode`, which allows you to override the implicit timeouts that exist in vttablet. However, this mode should be used judiciously as it supersedes shutdown and reparent commands.
-
-The general convention is to send OLTP queries to `REPLICA` tablet types, and OLAP queries to `RDONLY`.
-
-## Is there a list of supported/unsupported queries?
-
-Please see "SQL Syntax" under [MySQL Compatibility](../../reference/mysql-compatibility).
-
-## If I have a log of all queries from my app. Is there a way I can try them against Vitess to see how they’ll work?
-
-Yes. The [vtexplain](../../user-guides/sql/vtexplain) tool can be used to preview how your queries will be executed by Vitess. It can also be used to try different sharding scenarios before deciding on one.
diff --git a/content/en/docs/faq/vindexes.md b/content/en/docs/faq/vindexes.md
deleted file mode 100644
index 06ff4a894..000000000
--- a/content/en/docs/faq/vindexes.md
+++ /dev/null
@@ -1,9 +0,0 @@
----
-title: Vindexes
-description: Frequently Asked Questions about Vindexes
----
-
-
-## Does the Primary Vindex for a tablet have to be the same as its Primary Key?
-
-It is not necessary that a Primary Vindex be the same as the Primary Key. In fact, there are many use cases where you would not want this. For example, if there are tables with one-to-many relationships, the Primary Vindex of the main table is likely to be the same as the Primary Key. However, if you want the rows of the secondary table to reside in the same shard as the parent row, the Primary Vindex for that table must be the foreign key that points to the main table. A typical example is a user and order table. In this case, the order table has the `user_id` as a foreign key to the `id` of the user table. The `order_id` may be the primary key for `order`, but you may still want to choose `user_id` as Primary Vindex, which will make a user's orders live in the same shard as the user.
diff --git a/content/en/docs/get-started/helm.md b/content/en/docs/get-started/helm.md
index 82f06fda6..5b7b546c3 100644
--- a/content/en/docs/get-started/helm.md
+++ b/content/en/docs/get-started/helm.md
@@ -4,7 +4,7 @@ weight: 6
featured: true
---
-This tutorial is deprecated. We recommend that you use the [Operator](operator) instead.
+This tutorial is deprecated. We recommend that you use the [Operator](../operator) instead.
This tutorial demonstrates how Vitess can be used with Minikube to deploy Vitess clusters using Helm.
diff --git a/content/en/docs/get-started/local.md b/content/en/docs/get-started/local.md
index 2a29e89d0..621878792 100644
--- a/content/en/docs/get-started/local.md
+++ b/content/en/docs/get-started/local.md
@@ -61,10 +61,13 @@ sudo setenforce 0
Download the [latest binary release](https://github.com/vitessio/vitess/releases) for Vitess on Linux. For example with Vitess 6:
```sh
-tar -xzf vitess-6.0.20-20200508-147bc5a.tar.gz
-cd vitess-6.0.20-20200508-147bc5a
+version=6.0.20-20200818
+file=vitess-${version}-90741b8.tar.gz
+wget https://github.com/vitessio/vitess/releases/download/v${version}/${file}
+tar -xzf ${file}
+cd ${file/.tar.gz/}
sudo mkdir -p /usr/local/vitess
-sudo mv * /usr/local/vitess/
+sudo cp -r * /usr/local/vitess/
```
Make sure to add `/usr/local/vitess/bin` to the `PATH` environment variable. You can do this by adding the following to your `$HOME/.bashrc` file:
diff --git a/content/en/docs/overview/whatisvitess.md b/content/en/docs/overview/whatisvitess.md
index cd9b33734..ddb26659c 100644
--- a/content/en/docs/overview/whatisvitess.md
+++ b/content/en/docs/overview/whatisvitess.md
@@ -48,8 +48,8 @@ The following sections compare Vitess to two common alternatives, a vanilla MySQ
Vitess improves a vanilla MySQL implementation in several ways:
-| Vanilla MySQL | Vitess |
-|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Vanilla MySQL | Vitess |
+|:--|:--|
| Every MySQL connection has a memory overhead that ranges between 256KB and almost 3MB, depending on which MySQL release you're using. As your user base grows, you need to add RAM to support additional connections, but the RAM does not contribute to faster queries. In addition, there is a significant CPU cost associated with obtaining the connections. | Vitess creates very lightweight connections. Vitess' connection pooling feature uses Go's concurrency support to map these lightweight connections to a small pool of MySQL connections. As such, Vitess can easily handle thousands of connections. |
| Poorly written queries, such as those that don't set a LIMIT, can negatively impact database performance for all users. | Vitess employs a SQL parser that uses a configurable set of rules to rewrite queries that might hurt database performance. |
| Sharding is a process of partitioning your data to improve scalability and performance. MySQL lacks native sharding support, requiring you to write sharding code and embed sharding logic in your application. | Vitess supports a variety of sharding schemes. It can also migrate tables into different databases and scale up or down the number of shards. These functions are performed non-intrusively, completing most data transitions with just a few seconds of read-only downtime. |
@@ -61,8 +61,8 @@ Vitess improves a vanilla MySQL implementation in several ways:
If you're considering a NoSQL solution primarily because of concerns about the scalability of MySQL, Vitess might be a more appropriate choice for your application. While NoSQL provides great support for unstructured data, Vitess still offers several benefits not available in NoSQL datastores:
-| NoSQL | Vitess |
-|---------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------|
+| NoSQL | Vitess |
+|:--|:--|
| NoSQL databases do not define relationships between database tables, and only support a subset of the SQL language. | Vitess is not a simple key-value store. It supports complex query semantics such as where clauses, JOINS, aggregation functions, and more. |
| NoSQL datastores do not usually support transactions. | Vitess supports transactions. |
| NoSQL solutions have custom APIs, leading to custom architectures, applications, and tools. | Vitess adds very little variance to MySQL, a database that most people are already accustomed to working with. |
diff --git a/content/en/docs/pdfs/_index.md b/content/en/docs/pdfs/_index.md
index fdcae6546..e60958024 100644
--- a/content/en/docs/pdfs/_index.md
+++ b/content/en/docs/pdfs/_index.md
@@ -2,9 +2,14 @@
title: Older Version Docs
description: PDFs of vitess.io/docs at the time of previous versions
weight: 25
+docs_nav_disable_expand: true
---
-To view a PDF of the state of the Vitess Documentation for previous versions of Vitess please follow the links below:
+The Vitess Documentation currently posted should reflect the most recent code.
-- [Vitess Docs 5.0 on April 17, 2020](https://drive.google.com/file/d/1gK6ELHFxr5X9Rieg64XJ7iFcnOuUlwb-/view?usp=sharing)
-- [Vitess Docs 6.0 on July 28, 2020](https://drive.google.com/file/d/11r7trOcjjKnxcPJWa8Ae7lkti_Ny0HqV/view?usp=sharing)
\ No newline at end of file
+To view a PDF of the state of the Vitess Documentation on the date of the general availability release for previous versions please follow the links below:
+
+- [Vitess Docs 8.0 on October 27, 2020](https://drive.google.com/file/d/1SMnw19uYYFl9vqYsbc67jIC-8VObfiwh/view?usp=sharing)
+- [Vitess Docs 7.0 on July 28, 2020](https://drive.google.com/file/d/1Uu5JU-tOzElRbLSnOS2ZaTE8BOI6DAWG/view?usp=sharing)
+- [Vitess Docs 6.0 on April 29, 2020](https://drive.google.com/file/d/1b3EwRxYc3uBfnCKDvMhLXWEZenvDYX2u/view?usp=sharing)
+- [Vitess Docs 5.0 on February 4, 2020](https://drive.google.com/file/d/1RaWnhR7C2RyaaXY6_82V8Kb03JjC5FCD/view?usp=sharing)
diff --git a/content/en/docs/reference/compatibility/_index.md b/content/en/docs/reference/compatibility/_index.md
new file mode 100644
index 000000000..a41da89f9
--- /dev/null
+++ b/content/en/docs/reference/compatibility/_index.md
@@ -0,0 +1,6 @@
+---
+title: Compatibility
+description: Reference documents for MySQL compatibility with Vitess
+weight: 4
+---
+
diff --git a/content/en/docs/reference/mysql-compatibility.md b/content/en/docs/reference/compatibility/mysql-compatibility.md
similarity index 93%
rename from content/en/docs/reference/mysql-compatibility.md
rename to content/en/docs/reference/compatibility/mysql-compatibility.md
index c0a329dd8..3394d9379 100644
--- a/content/en/docs/reference/mysql-compatibility.md
+++ b/content/en/docs/reference/compatibility/mysql-compatibility.md
@@ -1,7 +1,7 @@
---
title: MySQL Compatibility
weight: 1
-aliases: ['/docs/reference/mysql-server-protocol/']
+aliases: ['/docs/reference/mysql-server-protocol/', '/docs/reference/mysql-compatibility/']
---
VTGate servers speak both gRPC and the MySQL server protocol. This allows you to connect to Vitess as if it were a MySQL Server without any changes to application code. This document refers to known compatibility issues where Vitess differs from MySQL.
@@ -16,7 +16,7 @@ The following describes some of the major differences in SQL Syntax handling bet
### DDL
-Vitess supports MySQL DDL, and will send `ALTER TABLE` statements to each of the underlying tablet servers. For large tables it is recommended to use an external schema deployment tool and apply directly to the underlying MySQL shard instances. This is discussed further in [Applying MySQL Schema](../../user-guides/operating-vitess/making-schema-changes).
+Vitess supports MySQL DDL, and will send `ALTER TABLE` statements to each of the underlying tablet servers. For large tables it is recommended to use an external schema deployment tool and apply directly to the underlying MySQL shard instances. This is discussed further in [Applying MySQL Schema](../../../user-guides/operating-vitess/making-schema-changes).
### Join Queries
@@ -44,7 +44,7 @@ Vitess does not yet support killing running shard queries via the `KILL` command
### Cross-shard Transactions
-By default, Vitess does not support transactions that span across shards. While Vitess can support this with the use of [Two-Phase Commit](../two-phase-commit), it is usually recommended to design the VSchema in such a way that cross-shard modifications are not required.
+By default, Vitess does not support transactions that span across shards. While Vitess can support this with the use of [Two-Phase Commit](../../features/two-phase-commit), it is usually recommended to design the VSchema in such a way that cross-shard modifications are not required.
### OLAP Workload
@@ -101,7 +101,7 @@ Vitess supports all of the data types available in MySQL. Using the `FLOAT` data
## Auto Increment
-Tables in sharded keyspaces do not support the `auto_increment` column attribute, as the values generated would be local only to each shard. [Vitess Sequences](../vitess-sequences) are provided as an alternative, which have very close semantics to `auto_increment`.
+Tables in sharded keyspaces do not support the `auto_increment` column attribute, as the values generated would be local only to each shard. [Vitess Sequences](../../features/vitess-sequences) are provided as an alternative, which have very close semantics to `auto_increment`.
## Extensions to MySQL Syntax
diff --git a/content/en/docs/reference/features/_index.md b/content/en/docs/reference/features/_index.md
index d20dc7715..6d3b5e10c 100644
--- a/content/en/docs/reference/features/_index.md
+++ b/content/en/docs/reference/features/_index.md
@@ -1,5 +1,6 @@
---
title: Features
description: Reference documents for Vitess features
+weight: 1
---
diff --git a/content/en/docs/reference/features/img/vschema_arch.png b/content/en/docs/reference/features/img/vschema_arch.png
new file mode 100644
index 000000000..ac1c9a969
Binary files /dev/null and b/content/en/docs/reference/features/img/vschema_arch.png differ
diff --git a/content/en/docs/reference/features/messaging.md b/content/en/docs/reference/features/messaging.md
index e7476ce01..2082699de 100644
--- a/content/en/docs/reference/features/messaging.md
+++ b/content/en/docs/reference/features/messaging.md
@@ -1,5 +1,6 @@
---
title: Messaging
+weight: 18
aliases: ['/docs/advanced/messaging/','/docs/reference/messaging/']
---
diff --git a/content/en/docs/reference/features/monitoring.md b/content/en/docs/reference/features/monitoring.md
new file mode 100644
index 000000000..858a8b6a6
--- /dev/null
+++ b/content/en/docs/reference/features/monitoring.md
@@ -0,0 +1,42 @@
+---
+title: Monitoring
+weight: 8
+---
+
+# Current state of monitoring
+
+There are currently three main ways that a Vitess cluster can be monitored. Depending on your needs, you can use any of the following methods:
+
+## 1. Vitess status pages
+
+The status HTML pages of various Vitess components can be accessed by pointing your browser to `http://:/debug/status`. The status pages will often display some basic, but useful, information for monitoring. For example, the status page of a vttablet will show the QPS graph for the past few minutes.
+
+Viewing a status page can be useful since it works out of the box, but it only provides very basic monitoring capabilities.
+
+## 2. Pull-based metrics system
+
+Vitess uses Go’s [expvar package](https://golang.org/pkg/expvar/) to expose various metrics, with the expectation that a user can configure a pull-based metrics system to ingest those metrics. Metrics are published to `http://:/debug/vars` as JSON key-value pairs, which should be easy for any metrics system to parse.
+
+Scraping Vitess variables is a good way to integrate Vitess into an existing monitoring system, and is useful for building up detailed monitoring dashboards. It is also the officially supported way for monitoring Vitess.
+
+## 3. Push-based metrics system
+
+Vitess also includes support for push-based metrics systems via plug-ins. Each Vitess component would need to be run with the `--emit_stats` flag.
+
+By default, the stats_emit_period is 60s, so each component will push stats to the selected backend every minute. This is configurable via the `--stats_emit_period` flag.
+
+Vitess has preliminary plug-ins to support OpenTSDB as a push-based metrics backend.
+
+It should be fairly straightforward to write your own plug-in, if you want to support a different backend. The plug-in package simply needs to implement the `PushBackend` interface of the `stats` package. For an example, you can see the [OpenTSDB plugin](https://github.com/vitessio/vitess/blob/master/go/stats/opentsdb/opentsdb.go).
+
+Once you’ve written the backend plug-in, you also need to register the plug-in from within all the relevant Vitess binaries. An example of how to do this can be seen in [this pull request](https://github.com/vitessio/vitess/pull/469).
+
+You can then specify that Vitess should publish stats to the backend that you’re targeting by using the `--stats_backend` flag.
+
+Connecting Vitess to a push-based metrics system can be useful if you’re already running a push-based system that you would like to integrate into.
+
+# Monitoring with Kubernetes
+
+The existing methods for integrating metrics are not supported in a Kubernetes environment by the Vitess team yet, but are on the roadmap for the future. However, it should be possible to get the Prometheus backend working with Kubernetes.
+
+In the meantime, if you run into issues or have questions, please post on our [Slack](https://vitess.io/slack).
diff --git a/content/en/docs/reference/features/mysql-replication.md b/content/en/docs/reference/features/mysql-replication.md
index 1e0a74154..3ee42df4c 100644
--- a/content/en/docs/reference/features/mysql-replication.md
+++ b/content/en/docs/reference/features/mysql-replication.md
@@ -1,6 +1,6 @@
---
title: Replication
-weight: 5
+weight: 9
aliases: ['/docs/reference/row-based-replication/','/docs/reference/vitess-replication/','/docs/reference/mysql-replication/']
---
@@ -18,7 +18,7 @@ Vitess strongly recommends the use of Semi-synchronous replication for High Avai
* Tablets of type rdonly will not send semi-sync ACKs. This is intentional because rdonly tablets are not eligible to be promoted to master, so Vitess avoids the case where a rdonly tablet is the single best candidate for election at the time of master failure.
-These behaviors combine to give you the property that, in case of master failure, there is at least one other replica that has every transaction that was ever reported to clients as having completed. You can then ([manually](../../vtctl/#emergencyreparentshard), or [using Orchestrator](../../../user-guides/configuration-advanced/integration-with-orchestrator/) to pick the replica that is farthest ahead in GTID position and promote that to be the new master.
+These behaviors combine to give you the property that, in case of master failure, there is at least one other replica that has every transaction that was ever reported to clients as having completed. You can then ([manually](../../programs/vtctl/#emergencyreparentshard), or [using Orchestrator](../../../user-guides/configuration-advanced/integration-with-orchestrator/) to pick the replica that is farthest ahead in GTID position and promote that to be the new master.
Thus, you can survive sudden master failure without losing any transactions that were reported to clients as completed. In MySQL 5.7+, this guarantee is strengthened slightly to preventing loss of any transactions that were ever **committed** on the original master, eliminating so-called [phantom reads](http://bugs.mysql.com/bug.php?id=62174).
diff --git a/content/en/docs/reference/features/recovery.md b/content/en/docs/reference/features/recovery.md
index 9a8c88506..ca411eaa5 100644
--- a/content/en/docs/reference/features/recovery.md
+++ b/content/en/docs/reference/features/recovery.md
@@ -1,5 +1,6 @@
---
title: Point In Time Recovery
+weight: 17
aliases: ['/docs/recovery/pitr','/docs/reference/pitr/']
---
diff --git a/content/en/docs/reference/features/schema-management.md b/content/en/docs/reference/features/schema-management.md
index 00eabfe23..77fd29d44 100644
--- a/content/en/docs/reference/features/schema-management.md
+++ b/content/en/docs/reference/features/schema-management.md
@@ -1,5 +1,6 @@
---
title: Schema Management
+weight: 16
aliases: ['/docs/schema-management/','/docs/user-guides/schema-management/','/docs/reference/schema-management/']
---
@@ -86,7 +87,7 @@ Vitess' schema modification functionality is designed the following goals in min
* Guarantee very little downtime (or no downtime) for most schema updates.
* Do not store permanent schema data in the topology service.
-Note that, at this time, Vitess only supports [data definition statements](https://dev.mysql.com/doc/refman/5.6/en/sql-syntax-data-definition.html) that create, modify, or delete database tables. For instance, `ApplySchema` does not affect stored procedures or grants.
+Note that, at this time, Vitess only supports [data definition statements](https://dev.mysql.com/doc/refman/5.6/en/sql-data-definition-statements.html) that create, modify, or delete database tables. For instance, `ApplySchema` does not affect stored procedures or grants.
The [ApplySchema](../../../reference/programs/vtctl/#applyvschema) command applies a schema change to the specified keyspace on every master tablet, running in parallel on all shards. Changes are then propagated to replicas. The command format is: `ApplySchema {-sql= || -sql_file=} `
@@ -131,4 +132,4 @@ The [`ApplyVSchema`](../../../reference/programs/vtctl/#applyvschema) command ap
### RebuildVSchemaGraph
-The [`RebuildVSchemaGraph`](../../../programs/vtctl/#rebuildvschemagraph) command propagates the global VSchema to a specific cell or the list of specified cells.
+The [`RebuildVSchemaGraph`](../../../reference/programs/vtctl/#rebuildvschemagraph) command propagates the global VSchema to a specific cell or the list of specified cells.
diff --git a/content/en/docs/reference/features/schema-routing-rules.md b/content/en/docs/reference/features/schema-routing-rules.md
index 706e691f2..49fd5a070 100644
--- a/content/en/docs/reference/features/schema-routing-rules.md
+++ b/content/en/docs/reference/features/schema-routing-rules.md
@@ -1,5 +1,6 @@
---
title: Schema Routing Rules
+weight: 15
aliases: ['/docs/schema-management/routing-rules/','/docs/reference/schema-routing-rules/']
---
diff --git a/content/en/docs/reference/features/sharding.md b/content/en/docs/reference/features/sharding.md
index 05aa11324..4100c7df7 100644
--- a/content/en/docs/reference/features/sharding.md
+++ b/content/en/docs/reference/features/sharding.md
@@ -1,7 +1,7 @@
---
title: Sharding
description: Shard widely, shard often.
-weight: 5
+weight: 1
aliases: ['/docs/sharding/','/user-guide/sharding.html','/docs/reference/sharding/']
---
@@ -94,5 +94,5 @@ Cool a hot tablet | For read access, add replicas or split shards. For write acc
Vitess provides the following tools to help manage range-based shards:
-* The [vtctl](../../vtctl) command-line tool supports functions for managing keyspaces, shards, tablets, and more.
+* The [vtctl](../../../reference/programs/vtctl) command-line tool supports functions for managing keyspaces, shards, tablets, and more.
* Client APIs account for sharding operations.
\ No newline at end of file
diff --git a/content/en/docs/reference/features/table-lifecycle.md b/content/en/docs/reference/features/table-lifecycle.md
index 729711891..745c79339 100644
--- a/content/en/docs/reference/features/table-lifecycle.md
+++ b/content/en/docs/reference/features/table-lifecycle.md
@@ -1,11 +1,12 @@
---
title: Table lifecycle
+weight: 22
aliases: ['/docs/user-guides/table-lifecycle/','/docs/reference/table-lifecycle/']
---
Vitess manages a table lifecycle flow, an abstraction and automation for a `DROP TABLE` operation.
-# Problems with DROP TABLE
+## Problems with DROP TABLE
Vitess inherits the same issues that MySQL has with `DROP TABLE`. Doing a direct
`DROP TABLE my_table` in production can be a risky operation. In busy environments
@@ -29,7 +30,7 @@ various factors:
It is common practice to avoid direct `DROP TABLE` statements and to follow
a more elaborate table lifecycle.
-# Vitess table lifecycle
+## Vitess table lifecycle
The lifecycle offered by Vitess consists of the following stages or some subset:
@@ -47,7 +48,7 @@ To understand the flow better, consider the following breakdown:
- `drop`: an actual `DROP TABLE` is imminent
- _removed_: table is dropped. When using InnoDB and `innodb_file_per_table` this means the `.ibd` data file backing the table is removed, and disk space is reclaimed.
-# Lifecycle subsets and configuration
+## Lifecycle subsets and configuration
Different environments and users have different requirements and workflows. For example:
@@ -61,10 +62,21 @@ Vitess will always work the steps in this order: `hold -> purge -> evac -> drop`
All subsets end with a `drop`, even if not explicitly mentioned. Thus, `"purge"` is interpreted as `"purge,drop"`.
-# Automated lifecycle
+## Stateless flow by table name hints
+
+Vitess does not track the state of the table lifecycle. The process is stateless thanks to an encoding scheme in the table names. Examples:
+
+- The table `_vt_HOLD_6ace8bcef73211ea87e9f875a4d24e90_20210915120000` is held until `2021-09-15 12:00:00`. The data remains intact.
+- The table `_vt_PURGE_6ace8bcef73211ea87e9f875a4d24e90_20210915123000` is at the state where it is being purged, or queued to be purged. Once it's fully purged (zero rows remain), it transitions to the next stage.
+- The table `_vt_EVAC_6ace8bcef73211ea87e9f875a4d24e90_20210918093000` is held until `2021-09-18 09:30:00`
+- The table `_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20210921170000` is eligible to be dropped on `2021-09-21 17:00:00`
+
+## Automated lifecycle
Vitess internally uses the above table lifecycle for [online, managed schema migrations](../../../user-guides/schema-changes/managed-online-schema-changes/). Online schema migration tools `gh-ost` and `pt-online-schema-change` create artifact tables or end with leftover tables: Vitess automatically collects those tables. The artifact or leftover tables are immediate moved to `purge` state. Depending on `-table_gc_lifecycle`, they may spend time in this state, getting purged, or immediately transitioned to the next state.
-# User-facing DROP TABLE lifecycle
+## User-facing DROP TABLE lifecycle
+
+When using an online `ddl_strategy`, a `DROP TABLE` is a [managed schema migration](../../../user-guides/schema-changes/managed-online-schema-changes/). It is internally replaced by a `RENAME TABLE` statement, renaming it into a `HOLD` state (e.g. `_vt_HOLD_6ace8bcef73211ea87e9f875a4d24e90_20210915120000`). It will then participate in the table lifecycle mechanism. If `table_gc_lifecycle` does not include the `hold` state, the table proceeds to transition to next included state.
-Table lifecycle is not yet available directly to the application user. Vitess will introduce a special syntax to allow users to indicate they want Vitess to manage a table's lifecycle.
+A multi-table `DROP TABLE` statement is converted to multiple single-table `DROP TABLE` statements, each to then convert to a `RENAME TABLE` statement.
diff --git a/content/en/docs/reference/features/tablet-throttler.md b/content/en/docs/reference/features/tablet-throttler.md
index 12d805fbf..3e2b4c938 100644
--- a/content/en/docs/reference/features/tablet-throttler.md
+++ b/content/en/docs/reference/features/tablet-throttler.md
@@ -1,5 +1,6 @@
---
title: Tablet throttler
+weight: 21
aliases: ['/docs/user-guides/tablet-throttler/','/docs/reference/tablet-throttler/']
---
diff --git a/content/en/docs/reference/features/topology-service.md b/content/en/docs/reference/features/topology-service.md
index b808b2874..04cffd15f 100644
--- a/content/en/docs/reference/features/topology-service.md
+++ b/content/en/docs/reference/features/topology-service.md
@@ -1,5 +1,6 @@
---
title: Topology Service
+weight: 20
aliases: ['/docs/user-guides/topology-service/','/docs/reference/topology-service/']
---
diff --git a/content/en/docs/reference/features/transport-security-model.md b/content/en/docs/reference/features/transport-security-model.md
index 87fa3b407..1b8365fd0 100644
--- a/content/en/docs/reference/features/transport-security-model.md
+++ b/content/en/docs/reference/features/transport-security-model.md
@@ -1,5 +1,6 @@
---
title: Transport Security Model
+weight: 12
aliases: ['/docs/user-guides/transport-security-model/','/docs/reference/transport-security-model/']
---
diff --git a/content/en/docs/reference/features/two-phase-commit.md b/content/en/docs/reference/features/two-phase-commit.md
index 42bd4ee2b..3b72abdfa 100644
--- a/content/en/docs/reference/features/two-phase-commit.md
+++ b/content/en/docs/reference/features/two-phase-commit.md
@@ -1,5 +1,6 @@
---
title: Two-Phase Commit
+weight: 11
aliases: ['/docs/launching/twopc/','/docs/reference/two-phase-commit/']
---
diff --git a/content/en/docs/reference/features/vindexes.md b/content/en/docs/reference/features/vindexes.md
index 583c12589..425473c43 100644
--- a/content/en/docs/reference/features/vindexes.md
+++ b/content/en/docs/reference/features/vindexes.md
@@ -1,5 +1,6 @@
---
title: Vindexes
+weight: 10
aliases: ['/docs/schema-management/consistent-lookup/','/docs/reference/vindexes/']
---
diff --git a/content/en/docs/reference/features/vreplication.md b/content/en/docs/reference/features/vreplication.md
index ccc044420..e295109f0 100644
--- a/content/en/docs/reference/features/vreplication.md
+++ b/content/en/docs/reference/features/vreplication.md
@@ -1,5 +1,6 @@
---
title: VReplication
+weight: 2
aliases: ['/docs/advanced/vreplication/','/user-guide/update-stream/','/docs/reference/vreplication/']
---
diff --git a/content/en/docs/reference/features/vschema.md b/content/en/docs/reference/features/vschema.md
index 376fec002..8645df00b 100644
--- a/content/en/docs/reference/features/vschema.md
+++ b/content/en/docs/reference/features/vschema.md
@@ -1,13 +1,36 @@
---
title: VSchema
+weight: 9
aliases: ['/docs/schema-management/vschema/','/docs/reference/vschema/']
---
-## VSchemas describe how to shard data
+## Overview
-VSchema stands for Vitess Schema. In contrast to a traditional database schema that contains metadata about tables, a VSchema contains metadata about how tables are organized across keyspaces and shards. Simply put, it contains the information needed to make Vitess look like a single database server.
+VSchema stands for Vitess Schema. It is an abstraction layer that presents a unified view of the underlying keyspaces and shards, and gives the semblance of a single MySQL server.
-For example, the VSchema will contain the information about the sharding key for a sharded table. When the application issues a query with a WHERE clause that references the key, the VSchema information will be used to route the query to the appropriate shard.
+For example, VSchema will contain the information about the sharding key for a sharded table. When the application issues a query with a WHERE clause that references the key, the VSchema information will be used to route the query to the appropriate shard.
+
+## Architecture
+
+The VSchema is specified on a per-keyspace basis. Additionally, a separate set of RoutingRules can be specified. This information is stored in the global topology. A vtctld `RebuildVSchemaGraph` command combines the RoutingRules and the per-keyspace VSchemas into a unified data structure called SrvVSchema, which is deployed into the topo of each cell. The VTGates consume this information, which they use for planning and routing queries from the application.
+
+
+
+## Database Access Model
+
+A Vitess Keyspace is the logical equivalent to a MySQL database. The usual syntax used to access databases in mysql work in Vitess as well. For example, you can connect to a specific database by specifying the database name in the connection parameter. You can also change the database you are connected to through the `use` statement. While connected to a database, you can access a table from a different keyspace by qualifying the table name in your query, like `select * from other_keyspace.table`.
+
+### Tablet Types
+
+Unlike MySQL, the vitess servers unify all types of mysql servers. You can ask vitess to target a specific tablet type by qualifying it as part of the database name. For example, to access replica tablets, you may specify the database name as `keyspace@replica`. This name can also be specified in the connection string. If no tablet type is specified in the database name, then the value specified in VTGate's `-default_tablet_type` flag is used.
+
+### Unspecified Mode
+
+Additionally, you can connect without specifying a database name and still access tables without qualifying them by their keyspace. If the table name is unique across all keyspaces, then VTGate automatically sends the query to the associated keyspace. Otherwise, it returns an error. This mode is useful if you start off with a single keyspace and plan on splitting it into multiple parts.
+
+You can still specify a tablet type for the unspecified mode. For example, you can connect to `@replica` if you want to access the replica tablets in unspecified mode.
+
+Some frameworks require you to specify an explicit database name while connecting. In order to make them work in unspecified mode, you can specify the database name as `@replica` or `@master` instead of a blank one.
## Sharded keyspaces require a VSchema
@@ -16,7 +39,7 @@ A VSchema is needed to tie together all the databases that Vitess manages. For a
If you have multiple unsharded keyspaces, you can still avoid defining a VSchema in one of two ways:
1. Connect to a keyspace and all queries are sent to it.
-2. Connect to Vitess without specifying a keyspace, but use qualified names for tables, like `keyspace.table` in your queries.
+2. Connect to Vitess without specifying a keyspace (unspecified mode), but use qualified names for tables, like `keyspace.table` in your queries.
However, once the setup exceeds the above complexity, VSchemas become a necessity. Vitess has a [working demo](https://github.com/vitessio/vitess/tree/master/examples/demo) of VSchemas.
@@ -45,9 +68,23 @@ Vitess allows you to create an unsharded table and deploy it into all shards of
Typically, such a table has a canonical source in an unsharded keyspace, and the copies in the sharded keyspace are kept up-to-date through VReplication.
-## Configuration
-The configuration of your VSchema reflects the desired sharding configuration for your database, including whether or not your tables are sharded and whether you want to implement a secondary Vindex.
+## Per-Keyspace VSchema
+
+The VSchema uses a flexible proto JSON format. Essentially, you can use `snake_case` or `camelCase` for the keys.
+
+The configuration of your VSchema reflects the desired sharding configuration for your database, including whether or not your tables are sharded and whether you want to implement a secondary Vindex.
+
+### Commands
+
+You can use the following commands for maintaining the VSchema:
+* `GetVSchema `
+* `ApplyVSchema {-vschema= || -vschema_file= || -sql= || -sql_file=} [-cells=c1,c2,...] [-skip_rebuild] [-dry-run] `
+* `RebuildVSchemaGraph [-cells=c1,c2,...]`
+* `GetSrvVSchema `
+* `DeleteSrvVSchema `
+
+In order to verify that a VTGate has loaded SrvVSchema correctly, you can visit the `/debug/vschema` URL on the VTGate's http port.
### Unsharded Table
@@ -123,7 +160,7 @@ create table user_seq(id int, next_id bigint, cache bigint, primary key(id)) com
insert into user_seq(id, next_id, cache) values(0, 1, 3);
```
-For the sequence table, `id` is always 0. `next_id` starts off as 1, and the cache is usually a medium-sized number like 1000. In our example, we are using a small number to showcase how it works.
+For the sequence table, `id` is always 0. `next_id` starts off as 1, and the cache is usually a medium-sized number like 1000. In our example, we are using a small number to demonstrate how it works.
VSchema:
@@ -156,13 +193,15 @@ VSchema:
],
"auto_increment": {
"column": "user_id",
- "sequence": "user_seq"
+ "sequence": "lookup.user_seq"
}
}
}
}
```
+If necessary, the reference to the sequence table `lookup.user_seq` can be escaped using backticks.
+
### Specifying A Secondary Vindex
The following snippet shows how to configure a [Secondary Vindex](../vindexes/#secondary-vindexes) that is backed by a lookup table. In this case, the lookup table is configured to be in the unsharded lookup keyspace:
@@ -222,6 +261,42 @@ To recap, a checklist for creating the shared Secondary Vindex is:
Currently, these steps have to be currently performed manually. However, extended DDLs backed by improved automation will simplify these tasks in the future.
+### The columns field
+
+For a table, you can specify an additional columns field. This can be used for two purposes:
+* Specifying that a column contains text. If so, the VTGate planner can rewrite queries to leverage mysql’s collation where possible.
+* If the full list of columns is specified, then VTGate can resolve columns to their tables where needed, and also authoritative expand column lists, like in the case of a `select *` or `insert` statements with no column list.
+
+Here is an example:
+
+``` json
+ "tables": {
+ "user": {
+ "column_vindexes": [
+ {
+ "column": "name",
+ "name": "name_user_idx"
+ }
+ ],
+ "columns": [
+ {
+ "name": "name",
+ "type": "VARCHAR"
+ },
+ {
+ "name": "keyspace_id",
+ "type": "VARBINARY"
+ }
+ ],
+ "column_list_authoritative": true
+ }
+ }
+```
+
+If a query goes across multiple shards and ordering is needed on the `name` column that is now specified as `VARCHAR`, then VTGate will leverage mysql to additionally the `weigh_string` of that column and use that value to order the merged results.
+
+If `column_list_authoritative` is false or not specified, then VTGate will treat the list of columns as partial and will not automatically expand open-ended constructs like `select *`.
+
### Advanced usage
The examples/demo also shows more tricks you can perform:
@@ -231,3 +306,65 @@ The examples/demo also shows more tricks you can perform:
* `music_extra` defines an additional Functional Vindex called `keyspace_id` which the demo auto-populates using the reverse mapping capability.
* There is also a `name_info` table that showcases a case-insensitive Vindex `unicode_loose_md5`.
+## Routing Rules
+
+The `RoutingRules` section of the VSchema can be used to dynamically route traffic of a tablet to a different table than originally referenced in the query. This feature is used by the `MoveTables` workflow allowing you to change the application independently of when the actual traffic is moved from the old source table to the new target table.
+
+Here is an example of `RoutingRules`
+
+``` json
+{
+ "rules": [
+ {
+ "from_table": "customer",
+ "to_tables": [
+ "commerce.customer"
+ ]
+ },
+ {
+ "from_table": "customer.customer",
+ "to_tables": [
+ "commerce.customer"
+ ]
+ },
+ {
+ "from_table": "customer.customer@replica",
+ "to_tables": [
+ "commerce.customer"
+ ]
+ }
+ ]
+}
+```
+
+In the above JSON data structure, each rule maps an input table to a target.
+If the input table name is unqualified, then any unqualified reference to that table gets redirected to the fully qualified `to_tables`.
+
+The `to_tables` field must contain only one entry and the table name must be fully qualified.
+
+If the `from_table` is qualified by a keyspace, then a query that references that table will get redirected to the corresponding target table. The reference need not be explicit. For example, if you are connected to the `customer` keyspace, then an unqualified reference to the `customer` table is interpreted as a qualified reference to `customer.customer`.
+
+You may further add a tablet type to the `from_table` field. If so, only queries that target that tablet type will get redirected. Although you can qualify a table by its keyspace in a query, there is no equivalent syntax for specifying the tablet type. The only way to choose a tablet type is through the `use` statement, like `use @replica`, or by specifying it in the connection string.
+
+The more specific rules supercede the less specific one. For example, `customer.customer@replica` is chosen over `customer.customer` if the current tablet type is a `replica`.
+
+If the `to_tables` have special characters that need escaping, you can use the mysql backtick syntax to do so. As for the `from_tables`, the table name should not be escaped. Instead, you should just concatenate the table with the keyspace without the backticks. In the following example, we are redirecting the `b.c` table to the `c.b` table in keyspace `a`:
+
+``` json
+{
+ "rules": [
+ {
+ "from_table": "a.b.c",
+ "to_tables": [
+ "a.`c.b`"
+ ]
+ }
+ ]
+}
+```
+
+### Commands
+
+You can use the following commands to maintain routing rules:
+* `GetRoutingRules`
+* `ApplyRoutingRules {-rules= || -rules_file=} [-cells=c1,c2,...] [-skip_rebuild] [-dry-run]`
diff --git a/content/en/docs/reference/programs/_index.md b/content/en/docs/reference/programs/_index.md
index 343c211c7..95f51ac2e 100644
--- a/content/en/docs/reference/programs/_index.md
+++ b/content/en/docs/reference/programs/_index.md
@@ -2,5 +2,6 @@
title: Programs
description: Reference documents for list of Vitess programs
notoc: true
+weight: 2
---
diff --git a/content/en/docs/reference/programs/vtctl.md b/content/en/docs/reference/programs/vtctl/_index.md
similarity index 97%
rename from content/en/docs/reference/programs/vtctl.md
rename to content/en/docs/reference/programs/vtctl/_index.md
index b3e891853..ff86d842c 100644
--- a/content/en/docs/reference/programs/vtctl.md
+++ b/content/en/docs/reference/programs/vtctl/_index.md
@@ -11,7 +11,7 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### Tablets
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [InitTablet](../vtctl/tablets#inittablet) | `InitTablet [-allow_update] [-allow_different_shard] [-allow_master_override] [-parent] [-db_name_override=] [-hostname=] [-mysql_port=] [-port=] [-grpc_port=] [-tags=tag1:value1,tag2:value2] -keyspace= -shard=` |
| [GetTablet](../vtctl/tablets#gettablet) | `GetTablet ` |
| [UpdateTabletAddrs](../vtctl/tablets#updatetabletaddrs) | `UpdateTabletAddrs [-hostname ] [-ip-addr ] [-mysql-port ] [-vt-port ] [-grpc-port ] ` |
@@ -19,11 +19,8 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
| [SetReadOnly](../vtctl/tablets#setreadonly) | `SetReadOnly ` |
| [SetReadWrite](../vtctl/tablets#setreadwrite) | `SetReadWrite ` |
| [StartReplication](../vtctl/tablets#startreplication) | `StartReplication ` |
-| [StartSlave](../vtctl/tablets#startreplication) | `DEPRECATED -- Use StartReplication ` |
| [StopReplication](../vtctl/tablets#stopreplication) | `StopReplication ` |
-| [StopSlave](../vtctl/tablets#stopreplication) | `DEPRECATED -- Use StopReplication ` |
| [ChangeTabletType](../vtctl/tablets#changetablettype) | `ChangeTabletType [-dry-run] ` |
-| [ChangeSlaveType](../vtctl/tablets#changetablettype) | `DEPRECATED -- Use ChangeTabletType [-dry-run] ` |
| [Ping](../vtctl/tablets#ping) | `Ping ` |
| [RefreshState](../vtctl/tablets#refreshstate) | `RefreshState ` |
| [RefreshStateByShard](../vtctl/tablets#refreshstatebyshard) | `RefreshStateByShard [-cells=c1,c2,...] ` |
@@ -41,7 +38,7 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### Shards
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [CreateShard](../vtctl/shards#createshard) | `CreateShard [-force] [-parent] ` |
| [GetShard](../vtctl/shards#getshard) | `GetShard ` |
| [ValidateShard](../vtctl/shards#validateshard) | `ValidateShard [-ping-tablets] ` |
@@ -67,7 +64,7 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### Keyspaces
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [CreateKeyspace](../vtctl/keyspaces#createkeyspace) | `CreateKeyspace [-sharding_column_name=name] [-sharding_column_type=type] [-served_from=tablettype1:ks1,tablettype2:ks2,...] [-force] [-keyspace_type=type] [-base_keyspace=base_keyspace] [-snapshot_time=time] ` |
| [DeleteKeyspace](../vtctl/keyspaces#deletekeyspace) | `DeleteKeyspace [-recursive] ` |
| [RemoveKeyspaceCell](../vtctl/keyspaces#removekeyspacesell) | `RemoveKeyspaceCell [-force] [-recursive] ` |
@@ -98,7 +95,7 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### Generic
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [Validate](../vtctl/generic#validate) | `Validate [-ping-tablets]` |
| [ListAllTablets](../vtctl/generic#listalltablets) | `ListAllTablets , , ...` |
| [ListTablets](../vtctl/generic#listtablets) | `ListTablets ...` |
@@ -107,7 +104,7 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### Schema, Version, Permissions
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [GetSchema](../vtctl/schema-version-permissions#getschema) | `GetSchema [-tables=,,...] [-exclude_tables=,,...] [-include-views] ` |
| [ReloadSchema](../vtctl/schema-version-permissions#reloadschema) | `ReloadSchema ` |
| [ReloadSchemaShard](../vtctl/schema-version-permissions#reloadschemashard) | `ReloadSchemaShard [-concurrency=10] [-include_master=false] ` |
@@ -130,7 +127,7 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### Serving Graph
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [GetSrvKeyspaceNames](../vtctl/serving-graph#getsrvkeyspacenames) | `GetSrvKeyspaceNames ` |
| [GetSrvKeyspace](../vtctl/serving-graph#getsrvkeyspace) | `GetSrvKeyspace ` |
| [GetSrvVSchema](../vtctl/serving-graph#getsrvsvchema) | `GetSrvVSchema ` |
@@ -139,13 +136,13 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### Replication Graph
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [GetShardReplication](../vtctl/replication-graph#getshardreplication) | `GetShardReplication ` |
### Cells
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [AddCellInfo](../vtctl/cells#addcellinfo) | `AddCellInfo [-server_address ] [-root ] ` |
| [UpdateCellInfo](../vtctl/cells#updatecellinfo) | `UpdateCellInfo [-server_address ] [-root ] ` |
| [DeleteCellInfo](../vtctl/cells#deletecellinfo) | `DeleteCellInfo [-force] ` |
@@ -155,7 +152,7 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### CellsAliases
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [AddCellsAlias](../vtctl/cell-aliases#addcellsalias) | `AddCellsAlias [-cells ] ` |
| [UpdateCellsAlias](../vtctl/cell-aliases#updatecellsalias) | `UpdateCellsAlias [-cells ] ` |
| [DeleteCellsAlias](../vtctl/cell-aliases#deletecellsalias) | `DeleteCellsAlias ` |
@@ -164,7 +161,7 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### Queries
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [VtGateExecute](../vtctl/queries#vtgateexecute) | `VtGateExecute -server [-bind_variables ] [-keyspace ] [-tablet_type ] [-options ] [-json] ` |
| [VtTabletExecute](../vtctl/queries#vttabletexecute) | `VtTabletExecute [-username ] [-transaction_id ] [-options ] [-json] ` |
| [VtTabletBegin](../vtctl/queries#vttabletbegin) | `VtTabletBegin [-username ] ` |
@@ -175,7 +172,7 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### Resharding Throttler
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [ThrottlerMaxRates](../vtctl/resharding-throttler#throttlermaxrates) | `ThrottlerMaxRates -server ` |
| [ThrottlerSetMaxRate](../vtctl/resharding-throttler#throttlersetmaxrate) | `ThrottlerSetMaxRate -server ` |
| [GetThrottlerConfiguration](../vtctl/resharding-throttler#getthrottlerconfiguration) | `GetThrottlerConfiguration -server []` |
@@ -185,14 +182,14 @@ aliases: ['/docs/reference/vitess-api/','/docs/reference/vtctl/']
### Topo
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [TopoCat](../vtctl/topo#topocat) | `TopoCat [-cell ] [-decode_proto] [-decode_proto_json] [-long] [...]` |
| [TopoCp](../vtctl/topo#topocp) | `TopoCp [-cell ] [-to_topo] ` |
### Workflows
| Name | Example Usage |
-| :-------- | :--------- |
+| :-------- | :--------------- |
| [WorkflowCreate](../vtctl/workflows#workflowcreate) | `WorkflowCreate [-skip_start] [parameters...]` |
| [WorkflowStart](../vtctl/workflows#workflowstart) | `WorkflowStart ` |
| [WorkflowStop](../vtctl/workflows#workflowstop) | `WorkflowStop ` |
@@ -207,8 +204,8 @@ The following global options apply to `vtctl`:
| Name | Type | Definition |
-| :-------- | :--------- | :--------- |
-| -alsologtostderr | | log to standard error as well as files|
+| :------------------------------------ | :--------- | :----------------------------------------------------------------------------------------- |
+| -alsologtostderr | | log to standard error as well as files |
| -app_idle_timeout | duration | Idle timeout for app connections (default 1m0s) |
| -app_pool_size | int | Size of the connection pool for app connections (default 40) |
| -azblob_backup_account_key_file | string | Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path) |
@@ -305,7 +302,7 @@ The following global options apply to `vtctl`:
| -pool_hostname_resolve_interval | duration | if set force an update to all hostnames and reconnect if changed, defaults to 0 (disabled) |
| -purge_logs_interval | duration | how often try to remove old logs (default 1h0m0s) |
| -query-log-stream-handler | string | URL handler for streaming queries log (default "/debug/querylog") |
-| -querylog-filter-tag | string | string that must be present in the query for it to be logged |
+| -querylog-filter-tag | string | string that must be present in the query as a comment for the query to be logged, works for both vtgate and vttablet |
| -querylog-format | string | format for query logs ("text" or "json") (default "text") |
| -queryserver-config-acl-exempt-acl | string | an acl that exempt from table acl checking (this acl is free to access any vitess tables). |
| -queryserver-config-enable-table-acl-dry-run | | If this flag is enabled, tabletserver will emit monitoring metrics and let the request pass regardless of table acl check results |
@@ -403,7 +400,7 @@ The following global options apply to `vtctl`:
| -twopc_abandon_age | float | time in seconds. Any unresolved transaction older than this time will be sent to the coordinator to be resolved. |
| -twopc_coordinator_address | string | address of the (VTGate) process(es) that will be used to notify of abandoned transactions. |
| -twopc_enable | | if the flag is on, 2pc is enabled. Other 2pc flags must be supplied.|
-| -tx-throttler-config | string | The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message (default "target_replication_lag_sec: 2\nmax_replication_lag_sec: 10\ninitial_rate: 100\nmax_increase: 1\nemergency_decrease: 0.5\nmin_duration_between_increases_sec: 40\nmax_duration_between_increases_sec: 62\nmin_duration_between_decreases_sec: 20\nspread_backlog_across_sec: 20\nage_bad_rate_after_sec: 180\nbad_rate_increase: 0.1\nmax_rate_approach_threshold: 0.9\n") |
+| -tx-throttler-config | string | The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message (default "target_replication_lag_sec: 2 max_replication_lag_sec: 10 initial_rate: 100 max_increase: 1 emergency_decrease: 0.5 min_duration_between_increases_sec: 40 max_duration_between_increases_sec: 62 min_duration_between_decreases_sec: 20 spread_backlog_across_sec: 20 age_bad_rate_after_sec: 180 bad_rate_increase: 0.1 max_rate_approach_threshold: 0.9 ") |
| -tx-throttler-healthcheck-cells | value | A comma-separated list of cells. Only tabletservers running in these cells will be monitored for replication lag by the transaction throttler. |
| -v | value | log level for V logs |
| -version | | print binary version |
@@ -434,4 +431,3 @@ The following global options apply to `vtctl`:
| -xtrabackup_stripe_block_size | uint | Size in bytes of each block that gets sent to a given stripe before rotating to the next stripe (default 102400) |
| -xtrabackup_stripes | uint | If greater than 0, use data striping across this many destination files to parallelize data transfer and decompression |
| -xtrabackup_user | string | User that xtrabackup will use to connect to the database server. This user must have all necessary privileges. For details, please refer to xtrabackup documentation. |
-
diff --git a/content/en/docs/reference/programs/vtctl/cell-aliases.md b/content/en/docs/reference/programs/vtctl/cell-aliases.md
index 6399d3e49..8c713b8fd 100644
--- a/content/en/docs/reference/programs/vtctl/cell-aliases.md
+++ b/content/en/docs/reference/programs/vtctl/cell-aliases.md
@@ -1,6 +1,7 @@
---
title: vtctl Cell Aliases Command Reference
series: vtctl
+docs_nav_title: Cell Aliases
---
The following `vtctl` commands are available for administering Cell Aliases.
diff --git a/content/en/docs/reference/programs/vtctl/cells.md b/content/en/docs/reference/programs/vtctl/cells.md
index 943cc9128..4b19debf5 100644
--- a/content/en/docs/reference/programs/vtctl/cells.md
+++ b/content/en/docs/reference/programs/vtctl/cells.md
@@ -1,6 +1,7 @@
---
title: vtctl Cell Command Reference
series: vtctl
+docs_nav_title: Cells
---
The following `vtctl` commands are available for administering Cells.
diff --git a/content/en/docs/reference/programs/vtctl/generic.md b/content/en/docs/reference/programs/vtctl/generic.md
index b26d5c206..bcd57bf6e 100644
--- a/content/en/docs/reference/programs/vtctl/generic.md
+++ b/content/en/docs/reference/programs/vtctl/generic.md
@@ -1,6 +1,7 @@
---
title: vtctl Generic Command Reference
series: vtctl
+docs_nav_title: Generic Commands
---
The following generic `vtctl` commands are available for administering Vitess.
diff --git a/content/en/docs/reference/programs/vtctl/keyspaces.md b/content/en/docs/reference/programs/vtctl/keyspaces.md
index cfac66cac..1e3de7015 100644
--- a/content/en/docs/reference/programs/vtctl/keyspaces.md
+++ b/content/en/docs/reference/programs/vtctl/keyspaces.md
@@ -1,6 +1,7 @@
---
title: vtctl Keyspace Command Reference
series: vtctl
+docs_nav_title: Keyspaces
---
The following `vtctl` commands are available for administering Keyspaces.
diff --git a/content/en/docs/reference/programs/vtctl/queries.md b/content/en/docs/reference/programs/vtctl/queries.md
index bbc1a67d7..51abac8e2 100644
--- a/content/en/docs/reference/programs/vtctl/queries.md
+++ b/content/en/docs/reference/programs/vtctl/queries.md
@@ -1,6 +1,7 @@
---
title: vtctl Query Command Reference
series: vtctl
+docs_nav_title: Queries
---
The following `vtctl` commands are available for administering queries.
diff --git a/content/en/docs/reference/programs/vtctl/replication-graph.md b/content/en/docs/reference/programs/vtctl/replication-graph.md
index a34742dd3..a4ab5b269 100644
--- a/content/en/docs/reference/programs/vtctl/replication-graph.md
+++ b/content/en/docs/reference/programs/vtctl/replication-graph.md
@@ -1,6 +1,7 @@
---
title: vtctl Replication Graph Command Reference
series: vtctl
+docs_nav_title: Replication Graph
---
The following `vtctl` commands are available for administering the Replication Graph.
diff --git a/content/en/docs/reference/programs/vtctl/resharding-throttler.md b/content/en/docs/reference/programs/vtctl/resharding-throttler.md
index b9c2392e0..6744fb4a1 100644
--- a/content/en/docs/reference/programs/vtctl/resharding-throttler.md
+++ b/content/en/docs/reference/programs/vtctl/resharding-throttler.md
@@ -1,6 +1,7 @@
---
title: vtctl Resharding Throttler Command Reference
series: vtctl
+docs_nav_title: Resharding Throttler
---
The following `vtctl` commands are available for administering Resharding Throttler.
diff --git a/content/en/docs/reference/programs/vtctl/schema-version-permissions.md b/content/en/docs/reference/programs/vtctl/schema-version-permissions.md
index 0ff17ff59..ec1258bc6 100644
--- a/content/en/docs/reference/programs/vtctl/schema-version-permissions.md
+++ b/content/en/docs/reference/programs/vtctl/schema-version-permissions.md
@@ -1,6 +1,7 @@
---
title: vtctl Schema, Version, Permissions Command Reference
series: vtctl
+docs_nav_title: Schema Versions & Permissions
---
The following `vtctl` commands are available for administering Schema, Versions and Permissions.
diff --git a/content/en/docs/reference/programs/vtctl/serving-graph.md b/content/en/docs/reference/programs/vtctl/serving-graph.md
index bbad0662e..0c756ce11 100644
--- a/content/en/docs/reference/programs/vtctl/serving-graph.md
+++ b/content/en/docs/reference/programs/vtctl/serving-graph.md
@@ -1,6 +1,7 @@
---
title: vtctl Serving Graph Command Reference
series: vtctl
+docs_nav_title: Serving Graph
---
The following `vtctl` commands are available for administering the Serving Graph.
diff --git a/content/en/docs/reference/programs/vtctl/shards.md b/content/en/docs/reference/programs/vtctl/shards.md
index a094898e1..1348473aa 100644
--- a/content/en/docs/reference/programs/vtctl/shards.md
+++ b/content/en/docs/reference/programs/vtctl/shards.md
@@ -1,6 +1,7 @@
---
title: vtctl Shard Command Reference
series: vtctl
+docs_nav_title: Shards
---
The following `vtctl` commands are available for administering shards.
@@ -400,7 +401,7 @@ Reparents the shard to the new master. Assumes the old master is dead and not re
### TabletExternallyReparented
-Changes metadata in the topology service to acknowledge a shard master change performed by an external tool. See [Reparenting](../../user-guides/reparenting/#external-reparenting) for more information.
+Changes metadata in the topology service to acknowledge a shard master change performed by an external tool. See [Reparenting](../../../../user-guides/configuration-advanced/reparenting/#external-reparenting) for more information.
#### Example
diff --git a/content/en/docs/reference/programs/vtctl/tablets.md b/content/en/docs/reference/programs/vtctl/tablets.md
index 4aa7c7070..aa1c24740 100644
--- a/content/en/docs/reference/programs/vtctl/tablets.md
+++ b/content/en/docs/reference/programs/vtctl/tablets.md
@@ -1,6 +1,7 @@
---
title: vtctl Tablet Command Reference
series: vtctl
+docs_nav_title: Tablets
---
The following `vtctl` commands are available for administering tablets.
diff --git a/content/en/docs/reference/programs/vtctl/topo.md b/content/en/docs/reference/programs/vtctl/topo.md
index 6a19f2723..856177f59 100644
--- a/content/en/docs/reference/programs/vtctl/topo.md
+++ b/content/en/docs/reference/programs/vtctl/topo.md
@@ -1,6 +1,7 @@
---
title: vtctl Topo Command Reference
series: vtctl
+docs_nav_title: Topology Service
---
The following `vtctl` commands are available for administering Topology Services.
diff --git a/content/en/docs/reference/programs/vtctl/workflows.md b/content/en/docs/reference/programs/vtctl/workflows.md
index 4abf898a5..b43f31aca 100644
--- a/content/en/docs/reference/programs/vtctl/workflows.md
+++ b/content/en/docs/reference/programs/vtctl/workflows.md
@@ -1,6 +1,7 @@
---
title: vtctl Workflow Command Reference
series: vtctl
+docs_nav_title: Workflows
---
The following `vtctl` commands are available for administering workflows.
diff --git a/content/en/docs/reference/programs/vtctld.md b/content/en/docs/reference/programs/vtctld.md
index 1e227261c..1b878ab34 100644
--- a/content/en/docs/reference/programs/vtctld.md
+++ b/content/en/docs/reference/programs/vtctld.md
@@ -29,7 +29,7 @@ vtctld \
## Options
| Name | Type | Definition |
-| :-------- | :--------- | :--------- |
+| :------------------------------------ | :--------- | :----------------------------------------------------------------------------------------- |
| -action_timeout | duration | time to wait for an action before resorting to force (default 2m0s) |
| -alsologtostderr | | log to standard error as well as files |
| -app_idle_timeout | duration | Idle timeout for app connections (default 1m0s) |
@@ -132,7 +132,7 @@ vtctld \
| -proxy_tablets | boolean | Setting this true will make vtctld proxy the tablet status instead of redirecting to them |
| -purge_logs_interval | duration | how often try to remove old logs (default 1h0m0s) |
| -query-log-stream-handler | string | URL handler for streaming queries log (default "/debug/querylog") |
-| -querylog-filter-tag | string | string that must be present in the query for it to be logged |
+| -querylog-filter-tag | string | string that must be present in the query as a comment for the query to be logged, works for both vtgate and vttablet |
| -querylog-format | string | format for query logs ("text" or "json") (default "text") |
| -queryserver-config-acl-exempt-acl | string | an acl that exempt from table acl checking (this acl is free to access any vitess tables). |
| -queryserver-config-enable-table-acl-dry-run | boolean | If this flag is enabled, tabletserver will emit monitoring metrics and let the request pass regardless of table acl check results |
@@ -173,7 +173,7 @@ vtctld \
| -schema_change_check_interval | int | this value decides how often we check schema change dir, in seconds (default 60) |
| -schema_change_controller | string | schema change controller is responsible for finding schema changes and responding to schema change events |
| -schema_change_dir | string | directory contains schema changes for all keyspaces. Each keyspace has its own directory and schema changes are expected to live in '$KEYSPACE/input' dir. e.g. test_keyspace/input/*sql, each sql file represents a schema change |
-| -schema_change_slave_timeout | duration | how long to wait for replicas to receive the schema change (default 10s) |
+| -schema_change_replicas_timeout | duration | how long to wait for replicas to receive the schema change (default 10s) |
| -schema_change_user | string | The user who submits this schema change. |
| -schema_swap_admin_query_timeout | duration | timeout for SQL queries used to save and retrieve meta information for schema swap process (default 30s) |
| -schema_swap_backup_concurrency | int | number of simultaneous compression/checksum jobs to run for seed backup during schema swap (default 4) |
@@ -238,7 +238,7 @@ vtctld \
| -twopc_abandon_age | float | time in seconds. Any unresolved transaction older than this time will be sent to the coordinator to be resolved. |
| -twopc_coordinator_address | string | address of the (VTGate) process(es) that will be used to notify of abandoned transactions. |
| -twopc_enable | boolean | if the flag is on, 2pc is enabled. Other 2pc flags must be supplied. |
-| -tx-throttler-config | string | The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message (default "target_replication_lag_sec: 2\nmax_replication_lag_sec: 10\ninitial_rate: 100\nmax_increase: 1\nemergency_decrease: 0.5\nmin_duration_between_increases_sec: 40\nmax_duration_between_increases_sec: 62\nmin_duration_between_decreases_sec: 20\nspread_backlog_across_sec: 20\nage_bad_rate_after_sec: 180\nbad_rate_increase: 0.1\nmax_rate_approach_threshold: 0.9\n") |
+| -tx-throttler-config | string | The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message (default "target_replication_lag_sec: 2 max_replication_lag_sec: 10 initial_rate: 100 max_increase: 1 emergency_decrease: 0.5 min_duration_between_increases_sec: 40 max_duration_between_increases_sec: 62 min_duration_between_decreases_sec: 20 spread_backlog_across_sec: 20 age_bad_rate_after_sec: 180 bad_rate_increase: 0.1 max_rate_approach_threshold: 0.9 ") |
| -tx-throttler-healthcheck-cells | value | A comma-separated list of cells. Only tabletservers running in these cells will be monitored for replication lag by the transaction throttler. |
| -v | value | log level for V logs |
| -version | | print binary version |
diff --git a/content/en/docs/reference/programs/vtgate.md b/content/en/docs/reference/programs/vtgate.md
index 0d1526f9a..e55c06c0d 100644
--- a/content/en/docs/reference/programs/vtgate.md
+++ b/content/en/docs/reference/programs/vtgate.md
@@ -32,7 +32,7 @@ vtgate \
The following global options apply to `vtgate`:
| Name | Type | Definition |
-| :-------- | :--------- | :--------- |
+| :------------------------------------ | :--------- | :----------------------------------------------------------------------------------------- |
| -allowed_tablet_types | value | Specifies the tablet types this vtgate is allowed to route queries to |
| -alsologtostderr | boolean | log to standard error as well as files |
| -buffer_drain_concurrency | int | Maximum number of requests retried simultaneously. More concurrency will increase the load on the MASTER vttablet when draining the buffer. (default 1) |
@@ -129,7 +129,7 @@ The following global options apply to `vtgate`:
| -port | int | port for the server |
| -proxy_protocol | boolean | Enable HAProxy PROXY protocol on MySQL listener socket |
| -purge_logs_interval | duration | how often try to remove old logs (default 1h0m0s) |
-| -querylog-filter-tag | string | string that must be present in the query for it to be logged |
+| -querylog-filter-tag | string | string that must be present in the query as a comment for the query to be logged, works for both vtgate and vttablet |
| -querylog-format | string | format for query logs ("text" or "json") (default "text") |
| -redact-debug-ui-queries | boolean | redact full queries and bind variables from debug UI |
| -remote_operation_timeout | duration | time to wait for a remote operation (default 30s) |
diff --git a/content/en/docs/reference/programs/vttablet.md b/content/en/docs/reference/programs/vttablet.md
index 317896f48..24f366948 100644
--- a/content/en/docs/reference/programs/vttablet.md
+++ b/content/en/docs/reference/programs/vttablet.md
@@ -8,7 +8,7 @@ A VTTablet server _controls_ a running MySQL server. VTTablet supports two prima
* Managed MySQL (most common)
* Unmanaged or Remote MySQL
-In addition to these deployment types, a partially managed VTTablet is also possible by setting `-disable_active_reparents`.
+In addition to these deployment types, a partially managed VTTablet is also possible by setting `-disable_active_reparents`.
## Example Usage
@@ -36,43 +36,19 @@ $TOPOLOGY_FLAGS
### Unmanaged or Remote MySQL
-In this mode, an external MySQL can be used such as RDS, Aurora, CloudSQL:
+In this mode, an external MySQL can be used such as AWS RDS, AWS Aurora, Google CloudSQL; or just an existing (vanilla) MySQL installation.
-```bash
-mkdir -p $VTDATAROOT/vt_0000000401
-vttablet \
- $TOPOLOGY_FLAGS \
- -logtostderr \
- -log_queries_to_file $VTDATAROOT/tmp/vttablet_0000000401_querylog.txt \
- -tablet-path "zone1-0000000401" \
- -init_keyspace legacy \
- -init_shard 0 \
- -init_tablet_type replica \
- -port 15401 \
- -grpc_port 16401 \
- -service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \
- -pid_file $VTDATAROOT/vt_0000000401/vttablet.pid \
- -vtctld_addr http://localhost:15000/ \
- -db_host 127.0.0.1 \
- -db_port 5726 \
- -db_app_user msandbox \
- -db_app_password msandbox \
- -db_dba_user msandbox \
- -db_dba_password msandbox \
- -db_repl_user msandbox \
- -db_repl_password msandbox \
- -db_filtered_user msandbox \
- -db_filtered_password msandbox \
- -db_allprivs_user msandbox \
- -db_allprivs_password msandbox \
- -init_db_name_override legacy \
- -init_populate_metadata &
+See [Unmanaged Tablet](../../../user-guides/configuration-advanced/unmanaged-tablet) for the full guide.
-sleep 10
-vtctlclient TabletExternallyReparented zone1-401
-```
+### Partially managed MySQL
-See [Unmanaged Tablet](../../../user-guides/configuration-advanced/unmanaged-tablet) for the full guide.
+Even if a MySQL is remote, you can still make vttablet perform some management functions. They are as follows:
+
+* `-disable_active_reparents`: If this flag is set, then any reparent or replica commands will not be allowed. These are InitShardMaster, PlannedReparent, PlannedReparent, EmergencyReparent, and ReparentTablet. In this mode, you should use the TabletExternallyReparented command to inform vitess of the current master.
+* `-master_connect_retry`: This value is give to mysql when it connects a replica to the master as the retry duration parameter.
+* `-enable_replication_reporter`: If this flag is set, then vttablet will transmit replica lag related information to the vtgates, which will allow it to balance load better. Additionally, enabling this will also cause vttablet to restart replication if it was stopped. However, it will do this only if -disable_active_reparents was not turned on.
+* `-enable_semi_sync`: This option will automatically enable semi-sync on new replicas as well as on any tablet that transitions into a replica type. This includes the demotion of a master to a replica.
+* `-heartbeat_enable` and `-heartbeat interval duration`: cause vttablet to write heartbeats to the sidecar database. This information is also used by the replication reporter to assess replica lag.
## Options
@@ -80,7 +56,7 @@ See [Unmanaged Tablet](../../../user-guides/configuration-advanced/unmanaged-tab
The following global options apply to `vttablet`:
| Name | Type | Definition |
-| :-------------------------------------------------- | :--------- | :-------------------------------------------------------------------------------------- |
+| :------------------------------------ | :--------- | :----------------------------------------------------------------------------------------- |
| -alsologtostderr | boolean | log to standard error as well as files |
| -app_idle_timeout | duration | Idle timeout for app connections (default 1m0s) |
| -app_pool_size | int | Size of the connection pool for app connections (default 40) |
@@ -255,7 +231,7 @@ The following global options apply to `vttablet`:
| -port | int | port for the server |
| -purge_logs_interval | duration | how often try to remove old logs (default 1h0m0s) |
| -query-log-stream-handler | string | URL handler for streaming queries log (default "/debug/querylog") |
-| -querylog-filter-tag | string | string that must be present in the query for it to be logged |
+| -querylog-filter-tag | string | string that must be present in the query as a comment for the query to be logged, works for both vtgate and vttablet |
| -querylog-format | string | format for query logs ("text" or "json") (default "text") |
| -queryserver-config-acl-exempt-acl | string | an acl that exempt from table acl checking (this acl is free to access any vitess tables). |
| -queryserver-config-enable-table-acl-dry-run | | If this flag is enabled, tabletserver will emit monitoring metrics and let the request pass regardless of table acl check results |
@@ -353,11 +329,11 @@ The following global options apply to `vttablet`:
| -transaction_limit_by_subcomponent | | Include CallerID.subcomponent when considering who the user is for the purpose of transaction limit. |
| -transaction_limit_by_username | | Include VTGateCallerID.username when considering who the user is for the purpose of transaction limit. (default true) |
| -transaction_limit_per_user | float | Maximum number of transactions a single user is allowed to use at any time, represented as fraction of -transaction_cap. (default 0.4) |
-| -transaction_shutdown_grace_period | int | how long to wait (in seconds) for transactions to complete during graceful shutdown. |
+| -shutdown_grace_period | float | how long to wait (in seconds) for queries and transactions to complete during graceful shutdown. |
| -twopc_abandon_age | float | time in seconds. Any unresolved transaction older than this time will be sent to the coordinator to be resolved. |
| -twopc_coordinator_address | string | address of the (VTGate) process(es) that will be used to notify of abandoned transactions. |
| -twopc_enable | | if the flag is on, 2pc is enabled. Other 2pc flags must be supplied. |
-| -tx-throttler-config | string | The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message (default "target_replication_lag_sec: 2\nmax_replication_lag_sec: 10\ninitial_rate: 100\nmax_increase: 1\nemergency_decrease: 0.5\nmin_duration_between_increases_sec: 40\nmax_duration_between_increases_sec: 62\nmin_duration_between_decreases_sec: 20\nspread_backlog_across_sec: 20\nage_bad_rate_after_sec: 180\nbad_rate_increase: 0.1\nmax_rate_approach_threshold: 0.9\n") |
+| -tx-throttler-config | string | The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message (default "target_replication_lag_sec: 2 max_replication_lag_sec: 10 initial_rate: 100 max_increase: 1 emergency_decrease: 0.5 min_duration_between_increases_sec: 40 max_duration_between_increases_sec: 62 min_duration_between_decreases_sec: 20 spread_backlog_across_sec: 20 age_bad_rate_after_sec: 180 bad_rate_increase: 0.1 max_rate_approach_threshold: 0.9 ") |
| -tx-throttler-healthcheck-cells | value | A comma-separated list of cells. Only tabletservers running in these cells will be monitored for replication lag by the transaction throttler. |
| -unhealthy_threshold | duration | replication lag after which a replica is considered unhealthy (default 2h0m0s) |
| -use_super_read_only | | Set super_read_only flag when performing planned failover. |
@@ -383,3 +359,10 @@ The following global options apply to `vttablet`:
| -xtrabackup_stripes | uint | If greater than 0, use data striping across this many destination files to parallelize data transfer and decompression |
| -xtrabackup_user | string | User that xtrabackup will use to connect to the database server. This user must have all necessary privileges. For details, please refer to xtrabackup documentation. |
+### Key Options
+
+* -restore_from_backup: The default value for this flag is false. If set to true, and the my.cnf file was successfully loaded, then vttablet can perform automatic restores as follows:
+
+ * If started against a mysql instance that has no data files, it will search the list of backups for the latest one, and initiate a restore. After this, it will point the mysql to the current master and wait for replication to catch up. Once replication is caught up to the specified tolerance limit, it will advertise itself as serving. This will cause the vtgates to add it to the list of healthy tablets to serve queries from.
+ * If this flag is true, but my.cnf was not loaded, then vttablet will fatally exit with an error message.
+ * You can additionally control the level of concurrency for a restore with the `-restore_concurrency` flag. This is typically useful in cloud environments to prevent the restore process from becoming a 'noisy' neighbor by consuming all available disk IOPS.
diff --git a/content/en/docs/reference/vreplication/_index.md b/content/en/docs/reference/vreplication/_index.md
index d2be8377c..21aab7f55 100644
--- a/content/en/docs/reference/vreplication/_index.md
+++ b/content/en/docs/reference/vreplication/_index.md
@@ -1,5 +1,6 @@
---
title: VReplication
description: Command references, architecture and design docs
+weight: 3
---
diff --git a/content/en/docs/reference/vreplication/internals.md b/content/en/docs/reference/vreplication/internals.md
index 48cb550db..988fc7752 100644
--- a/content/en/docs/reference/vreplication/internals.md
+++ b/content/en/docs/reference/vreplication/internals.md
@@ -33,7 +33,7 @@ the target. Columns may also be transformed in the Filter’s select clause.
#### Source and Sink
Each stream has two parts. The target initiates streaming by making grpc calls to the source tablet. The source
-sources the data connecting to mysql as a slave or using sql queries and streams it to the target. The target
+sources the data connecting to mysql as a replica or using sql queries and streams it to the target. The target
takes appropriate action: in case of resharding it will convert the events into CRUDs and apply it to the
target database. In case of vstream clients the events are forwarded by vtgate to the client.
@@ -48,7 +48,7 @@ of events can be substantially cpu intensive especially in the case of bulk inse
#### Replicate
-This is the easiest step to understand. The source stream just mimics a mysql slave and processes events as
+This is the easiest step to understand. The source stream just mimics a mysql replica and processes events as
they are received. Events (after filtering and transformation) are sent to the target. Replication runs continuously
with short sleeps when there are no more events to source.
diff --git a/content/en/docs/reference/vreplication/movetables.md b/content/en/docs/reference/vreplication/movetables.md
index 988848050..88a83c80a 100644
--- a/content/en/docs/reference/vreplication/movetables.md
+++ b/content/en/docs/reference/vreplication/movetables.md
@@ -13,7 +13,7 @@ MoveTables [-cells=] [-tablet_types=] -workflow=
-One of
+_Either_
+
+* a comma separated list of tables
+ * if target keyspace is unsharded OR
+ * if target keyspace is sharded AND the tables being moved are already defined in the target's vschema
+
+ Example: `MoveTables -workflow=commerce2customer commerce customer customer,corder`
+
+_Or_
+* the JSON table section of the vschema for associated tables
+ * if target keyspace is sharded AND
+ * tables being moved are not yet present in the target's vschema
+
+ Example: `MoveTables -workflow=commerce2customer commerce customer '{"t1":{"column_vindexes": [{"column": "id", "name": "hash"}]}}}'`
-* comma separated list of tables (if vschema has been already specified for all the tables)
-* JSON table section of the vschema for associated tables in case vschema is not yet specified
### A MoveTables Workflow
@@ -94,23 +105,19 @@ Once you select the set of tables to move from one keyspace to another you need
#### Adopting Vitess
For those wanting to try out Vitess for the first time MoveTables provides an easy way to route part of their workload
-to Vitess with the ability of migrating back at any time without any risk. You point a vttablet to your existing MySQL installation,
+to Vitess with the ability of migrating back at any time without any risk. You point a vttablet to your existing MySQL installation,
spin up a unsharded Vitess cluster and use a MoveTables workflow to start serving some tables from Vitess. You can also go
further and use a Reshard workflow to experiment with a sharded version of a part of your database.
-See [user guide](docs/user-guides/unmanaged-tablet/#move-legacytable-to-the-commerce-keyspace) for detailed steps
+See [user guide](../../../../../docs/user-guides/configuration-advanced/unmanaged-tablet/#move-legacytable-to-the-commerce-keyspace) for detailed steps
#### Horizontal Sharding
For existing Vitess users you can easily move one or more tables to another keyspace, either for balancing load or
as a preparation to sharding your tables.
-See [user guide](/docs/user-guides/move-tables/) which describes how MoveTables works in the local example provided
+See [user guide](../../../../../docs/user-guides/migration/move-tables/) which describes how MoveTables works in the local example provided
in the Vitess repo.
#### More Reading
-
-* [MoveTables in practice](docs/concepts/move-tables/)
-
-
-
+* [MoveTables in practice](../../../../../docs/concepts/move-tables/)
diff --git a/content/en/docs/resources/presentations.md b/content/en/docs/resources/presentations.md
index 38c863f30..1632acd2a 100644
--- a/content/en/docs/resources/presentations.md
+++ b/content/en/docs/resources/presentations.md
@@ -16,16 +16,22 @@ Lizz van Dijk presents an introduction to Vitess for MySQL users.
+## QCon 2019
+
+Sugu Sougoumarane gives an overview of the salient features of Vitess.
+
+[Video and Slides](https://www.infoq.com/presentations/vitess/)
+
## KubeCon San Diego 2019
KubeCon featured several Vitess talks, including:
-* [Scaling Resilient Systems: A Journey into Slack's Database Service](https://kccncna19.sched.com/event/UaWu/scaling-resilient-systems-a-journey-into-slacks-database-service-rafael-chacon-guido-iaquinti-slack) - Rafael Chacon & Guido Iaquinti, Slack
-* [How to Migrate a MySQL Database to Vitess](https://kccncna19.sched.com/event/UaiN/how-to-migrate-a-mysql-database-to-vitess-sugu-sougoumarane-morgan-tocker-planetscale) - Sugu Sougoumarane & Morgan Tocker, PlanetScale
-* [Building a Database as a Service on Kubernetes](https://kccncna19.sched.com/event/Uabw/building-a-database-as-a-service-on-kubernetes-abhi-vaidyanatha-lucy-burns-planetscale) - Abhi Vaidyanatha & Lucy Burns, PlanetScale
-* [Vitess: Stateless Storage in the Cloud](https://kccncna19.sched.com/event/UaeD/vitess-stateless-storage-in-the-cloud-sugu-sougoumarane-planetscale) - Sugu Sougoumarane, PlanetScale
-* [Geo-partitioning with Vitess](https://kccncna19.sched.com/event/UagI/geo-partitioning-with-vitess-deepthi-sigireddi-jitendra-vaidya-planetscale) - Deepthi Sigireddi & Jitendra Vaidya, PlanetScale
-* [Gone in 60 Minutes: Migrating 20 TB from AKS to GKE in an Hour with Vitess](https://kccncna19.sched.com/event/UaYn/gone-in-60-minutes-migrating-20-tb-from-aks-to-gke-in-an-hour-with-vitess-derek-perkins-nozzle) - Derek Perkins, Nozzle
+* [Scaling Resilient Systems: A Journey into Slack's Database Service](https://youtu.be/aTItjMJE17c) - Rafael Chacon & Guido Iaquinti, Slack
+* [How to Migrate a MySQL Database to Vitess](https://youtu.be/OCS45iy5v1M) - Sugu Sougoumarane & Morgan Tocker, PlanetScale
+* [Building a Database as a Service on Kubernetes](https://youtu.be/469NOldFOgw) - Abhi Vaidyanatha & Lucy Burns, PlanetScale
+* [Vitess: Stateless Storage in the Cloud](https://youtu.be/z63dtNj6ctY) - Sugu Sougoumarane, PlanetScale
+* [Geo-partitioning with Vitess](https://youtu.be/-Hz6LFJu1cY) - Deepthi Sigireddi & Jitendra Vaidya, PlanetScale
+* [Gone in 60 Minutes: Migrating 20 TB from AKS to GKE in an Hour with Vitess](https://youtu.be/KpygSD-v_ws) - Derek Perkins, Nozzle
Vitess was also featured during the [CNCF project updates keynote](https://www.youtube.com/watch?v=two3TzF9mVY&feature=youtu.be&t=105)!
@@ -188,8 +194,9 @@ Vitess team member [Anthony Yeh](https://github.com/enisoc)'s talk at
the [January 2016 CoreOS Meetup](http://www.meetup.com/coreos/events/228233948/)
discussed challenges and techniques for running distributed databases
within Kubernetes, followed by a deep dive into the design trade-offs
-of the [Vitess on Kubernetes](https://github.com/vitessio/vitess/tree/master/examples/kubernetes)
-deployment templates.
+of the Vitess on Kubernetes deployment templates.
+
+Please note the Vitess on Kubernetes deployment templates were removed as of February 27, 2020.
{{< pdf src="/ViewerJS/#../files/coreos-meetup-2016-01-27.pdf" >}}
diff --git a/content/en/docs/troubleshoot/_index.md b/content/en/docs/troubleshoot/_index.md
index b0f21c001..9cb6d54c9 100644
--- a/content/en/docs/troubleshoot/_index.md
+++ b/content/en/docs/troubleshoot/_index.md
@@ -14,28 +14,7 @@ When an alert fires, you have the following sources of information to perform yo
* Diagnostic URLs
* Log files
-Below are a few possible scenarios.
-
-## Elevated query latency on master
-
-Diagnosis 1: Inspect the graphs to see if QPS has gone up. If yes, drill down on the more detailed QPS graphs to see which table, or user caused the increase. If a table is identified, look at /debug/queryz for queries on that table.
-
-Action: Inform engineer about about toxic query. If it’s a specific user, you can stop their job or throttle them to keep the load manageable. As a last resort, blacklist query to allow the rest of the system to stay healthy.
-
-Diagnosis 2: QPS did not go up, only latency did. Inspect the per-table latency graphs. If it’s a specific table, then it’s most likely a long-running low QPS query that’s skewing the numbers. Identify the culprit query and take necessary steps to get it optimized. Such queries usually do not cause outage. So, there may not be a need to take extreme measures.
-
-Diagnosis 3: Latency seems to be up across the board. Inspect transaction latency. If this has gone up, then something is causing MySQL to run too many concurrent transactions which causes slow-down. See if there are any tx pool full errors. If there is an increase, the INFO logs will dump info about all transactions. From there, you should be able to if a specific sequence of statements is causing the problem. Once that is identified, find out the root cause. It could be network issues, or it could be a recent change in app behavior.
-
-Diagnosis 4: No particular transaction seems to be the culprit. Nothing seems to have changed in any of the requests. Look at system variables to see if there are hardware faults. Is the disk latency too high? Are there memory parity errors? If so, you may have to failover to a new machine.
-
-## Master starts up read-only
-
-To prevent accidentally accepting writes, our default my.cnf settings tell MySQL to always start up read-only. If the master MySQL gets restarted, it will thus come back read-only until you intervene to confirm that it should accept writes. You can use the [`SetReadWrite`](../reference/vtctl/#setreadwrite) command to do that.
-
-However, usually if something unexpected happens to the master, it's better to reparent to a different replica with [`EmergencyReparentShard`](../reference/vtctl/#emergencyreparentshard). If you need to do planned maintenance on the master, it's best to first reparent to another replica with [`PlannedReparentShard`](../reference/vtctl/#plannedreparentshard).
-
-## Vitess sees the wrong tablet as master
-
-If you do a failover manually (not through Vitess), you'll need to tell Vitess which tablet corresponds to the new master MySQL. Until then, writes will fail since they'll be routed to a read-only replica (the old master). Use the [`TabletExternallyReparented`](../reference/vtctl/#tabletexternallyreparented) command to tell Vitess the new master tablet for a shard.
-
-Tools like [Orchestrator](https://github.com/github/orchestrator) can be configured to call this automatically when a failover occurs. See our sample [orchestrator.conf.json](https://github.com/vitessio/vitess/blob/1129d69282bb738c94b8af661b984b6377a759f7/docker/orchestrator/orchestrator.conf.json#L131) for an example of this.
+### Find Vitess build running
+```
+select @@vitess_version;
+```
diff --git a/content/en/docs/troubleshoot/elevated-query-latency.md b/content/en/docs/troubleshoot/elevated-query-latency.md
new file mode 100644
index 000000000..12fd37cce
--- /dev/null
+++ b/content/en/docs/troubleshoot/elevated-query-latency.md
@@ -0,0 +1,16 @@
+---
+title: Elevated query latency on master
+description: Debug common issues with Vitess
+weight: 1
+---
+## Elevated query latency on master
+
+Diagnosis 1: Inspect the graphs to see if QPS has gone up. If yes, drill down on the more detailed QPS graphs to see which table, or user caused the increase. If a table is identified, look at /debug/queryz for queries on that table.
+
+Action: Inform engineer about about toxic query. If it’s a specific user, you can stop their job or throttle them to keep the load manageable. As a last resort, blacklist query to allow the rest of the system to stay healthy.
+
+Diagnosis 2: QPS did not go up, only latency did. Inspect the per-table latency graphs. If it’s a specific table, then it’s most likely a long-running low QPS query that’s skewing the numbers. Identify the culprit query and take necessary steps to get it optimized. Such queries usually do not cause outage. So, there may not be a need to take extreme measures.
+
+Diagnosis 3: Latency seems to be up across the board. Inspect transaction latency. If this has gone up, then something is causing MySQL to run too many concurrent transactions which causes slow-down. See if there are any tx pool full errors. If there is an increase, the INFO logs will dump info about all transactions. From there, you should be able to if a specific sequence of statements is causing the problem. Once that is identified, find out the root cause. It could be network issues, or it could be a recent change in app behavior.
+
+Diagnosis 4: No particular transaction seems to be the culprit. Nothing seems to have changed in any of the requests. Look at system variables to see if there are hardware faults. Is the disk latency too high? Are there memory parity errors? If so, you may have to failover to a new machine.
diff --git a/content/en/docs/troubleshoot/master-read-only.md b/content/en/docs/troubleshoot/master-read-only.md
new file mode 100644
index 000000000..124e93756
--- /dev/null
+++ b/content/en/docs/troubleshoot/master-read-only.md
@@ -0,0 +1,12 @@
+---
+title: Master starts up read-only
+description: Debug common issues with Vitess
+weight: 5
+---
+
+## Master starts up read-only
+
+To prevent accidentally accepting writes, our default my.cnf settings tell MySQL to always start up read-only. If the master MySQL gets restarted, it will thus come back read-only until you intervene to confirm that it should accept writes. You can use the [`SetReadWrite`](../../reference/programs/vtctl/#setreadwrite) command to do that.
+
+However, usually if something unexpected happens to the master, it's better to reparent to a different replica with [`EmergencyReparentShard`](../../reference/programs/vtctl/#emergencyreparentshard). If you need to do planned maintenance on the master, it's best to first reparent to another replica with [`PlannedReparentShard`](../../reference/programs/vtctl/#plannedreparentshard).
+
diff --git a/content/en/docs/troubleshoot/wrong-tablet-master.md b/content/en/docs/troubleshoot/wrong-tablet-master.md
new file mode 100644
index 000000000..d534dfbae
--- /dev/null
+++ b/content/en/docs/troubleshoot/wrong-tablet-master.md
@@ -0,0 +1,11 @@
+---
+title: Vitess sees the wrong tablet as master
+description: Debug common issues with Vitess
+weight: 10
+---
+
+## Vitess sees the wrong tablet as master
+
+If you do a failover manually (not through Vitess), you'll need to tell Vitess which tablet corresponds to the new master MySQL. Until then, writes will fail since they'll be routed to a read-only replica (the old master). Use the [`TabletExternallyReparented`](../../reference/programs/vtctl/#tabletexternallyreparented) command to tell Vitess the new master tablet for a shard.
+
+Tools like [Orchestrator](https://github.com/github/orchestrator) can be configured to call this automatically when a failover occurs. See our sample [orchestrator.conf.json](https://github.com/vitessio/vitess/blob/1129d69282bb738c94b8af661b984b6377a759f7/docker/orchestrator/orchestrator.conf.json#L131) for an example of this.
diff --git a/content/en/docs/user-guides/_index.md b/content/en/docs/user-guides/_index.md
index 466ca38c2..32844607d 100644
--- a/content/en/docs/user-guides/_index.md
+++ b/content/en/docs/user-guides/_index.md
@@ -2,7 +2,6 @@
title: User Guides
description: Task-based guides for common usage scenarios
weight: 4
-skip_sections: true
aliases: ['/user-guide/client-libraries.html', '/docs/user-guides/']
---
diff --git a/content/en/docs/user-guides/configuration-advanced/_index.md b/content/en/docs/user-guides/configuration-advanced/_index.md
index 22409a173..93e1c5d63 100644
--- a/content/en/docs/user-guides/configuration-advanced/_index.md
+++ b/content/en/docs/user-guides/configuration-advanced/_index.md
@@ -1,5 +1,5 @@
---
title: Advanced Configuration
description: User guides covering advanced configuration concepts
-weight: 3
----
\ No newline at end of file
+weight: 5
+---
diff --git a/content/en/docs/user-guides/configuration-advanced/authorization.md b/content/en/docs/user-guides/configuration-advanced/authorization.md
index 832f850e9..f7c5e8f52 100644
--- a/content/en/docs/user-guides/configuration-advanced/authorization.md
+++ b/content/en/docs/user-guides/configuration-advanced/authorization.md
@@ -1,6 +1,7 @@
---
title: Authorization
-weight: 2
+weight: 3
+aliases: ['/docs/user-guides/authorization/']
---
A common question is how to enforce fine-grained access control in Vitess.
diff --git a/content/en/docs/user-guides/configuration-advanced/createlookupvindex.md b/content/en/docs/user-guides/configuration-advanced/createlookupvindex.md
index 1f6cb9e20..2494d3dc0 100644
--- a/content/en/docs/user-guides/configuration-advanced/createlookupvindex.md
+++ b/content/en/docs/user-guides/configuration-advanced/createlookupvindex.md
@@ -1,10 +1,11 @@
---
title: CreateLookupVindex
weight: 11
+aliases: ['/docs/user-guides/createlookupvindex/']
---
{{< info >}}
-This guide follows on from the Get Started guides. Please make sure that you have an [Operator](../../../get-started/operator), [local](../../../get-started/local) or [Helm](../../../get-started/helm) installation ready. Make sure you are at the point where you have the sharded keyspace called `customer` setup.
+This guide follows on from the Get Started guides. Please make sure that you have an [Operator](../../../get-started/operator) or [local](../../../get-started/local) installation ready. Make sure you are at the point where you have the sharded keyspace called `customer` setup.
{{< /info >}}
**CreateLookupVindex** is a new VReplication workflow in Vitess 6. It is used to create **and** backfill a lookup Vindex automatically for a table that already exists, and may have a significant amount of data in it already.
@@ -323,18 +324,13 @@ mysql> select sku, hex(keyspace_id) from corder_lookup;
+-----------+------------------+
```
-Basically, this shows exactly what we expected. Now, we can clean up the
-VReplication streams. Note these commands will clean up all VReplication
-streams on these tablets. You may want to filter by `id` if there are other
-streams running:
+Basically, this shows exactly what we expected. Now, we have to clean-up
+the artifacts of the backfill. The `ExternalizeVindex` command will delete
+the vreplication streams and also clear the `write_only` flag from the
+vindex indicating that it is not backfilling any more.
```sh
-$ vtctlclient -server localhost:15999 VReplicationExec zone1-0000000300 "delete from _vt.vreplication"
-+
-+
-$ vtctlclient -server localhost:15999 VReplicationExec zone1-0000000400 "delete from _vt.vreplication"
-+
-+
+$ vtctlclient -server localhost:15999 ExternalizeVindex customer.corder_lookup
```
Next, to confirm the lookup Vindex is doing what we think it should, we can
@@ -474,3 +470,6 @@ mysql> select sku, hex(keyspace_id) from corder_lookup;
We added a new row to the `corder` table, and now we have a new row in the
lookup table.
+### ExternalizeVindex
+
+Once the backfill is done,
diff --git a/content/en/docs/user-guides/configuration-advanced/integration-with-orchestrator.md b/content/en/docs/user-guides/configuration-advanced/integration-with-orchestrator.md
index 516b47c0d..5efcb7c45 100644
--- a/content/en/docs/user-guides/configuration-advanced/integration-with-orchestrator.md
+++ b/content/en/docs/user-guides/configuration-advanced/integration-with-orchestrator.md
@@ -1,6 +1,7 @@
---
title: Integration with Orchestrator
-weight: 4
+weight: 5
+aliases: ['/docs/user-guides/integration-with-orchestrator/']
---
[Orchestrator](https://github.com/github/orchestrator) is a tool for managing MySQL replication topologies, including automated failover. It can detect master failure and initiate a recovery in a matter of seconds.
diff --git a/content/en/docs/user-guides/configuration-advanced/ldap_auth.md b/content/en/docs/user-guides/configuration-advanced/ldap_auth.md
index 898b3afaa..ae4b67177 100644
--- a/content/en/docs/user-guides/configuration-advanced/ldap_auth.md
+++ b/content/en/docs/user-guides/configuration-advanced/ldap_auth.md
@@ -1,6 +1,7 @@
---
title: LDAP authentication
-weight: 8
+weight: 2
+aliases: ['/docs/user-guides/ldap_auth/']
---
Currently, Vitess supports two ways to authenticate to `vtgate` via the MySQL protocol:
diff --git a/content/en/docs/user-guides/configuration-advanced/region-sharding.md b/content/en/docs/user-guides/configuration-advanced/region-sharding.md
index 6c690a4ae..d696a9249 100644
--- a/content/en/docs/user-guides/configuration-advanced/region-sharding.md
+++ b/content/en/docs/user-guides/configuration-advanced/region-sharding.md
@@ -1,6 +1,7 @@
---
title: Region-based Sharding
weight: 10
+aliases: ['/docs/user-guides/region-sharding/']
---
{{< info >}}
@@ -282,7 +283,7 @@ Once the vindex is available, we have to `Externalize` it for it to be usable.
Putting this all together, we run the script that combines the above steps.
```sh
-./201_initial_cluster.sh
+./201_main_sharded.sh
```
Once this is complete, we can view the new vschema. Note that it now includes both region_vdx and a lookup vindex.
```text
diff --git a/content/en/docs/user-guides/configuration-advanced/reparenting.md b/content/en/docs/user-guides/configuration-advanced/reparenting.md
index 14ae1b52c..6ce31ef39 100644
--- a/content/en/docs/user-guides/configuration-advanced/reparenting.md
+++ b/content/en/docs/user-guides/configuration-advanced/reparenting.md
@@ -1,7 +1,7 @@
---
title: Reparenting
-weight: 3
-aliases: ['/user-guide/reparenting.html','/user-guide/reparenting/']
+weight: 6
+aliases: ['/docs/user-guides/reparenting/']
---
**Reparenting** is the process of changing a shard's master tablet from one host to another or changing a replica tablet to have a different master. Reparenting can be initiated manually or it can occur automatically in response to particular database conditions. As examples, you might reparent a shard or tablet during a maintenance exercise or automatically trigger reparenting when a master tablet dies.
diff --git a/content/en/docs/user-guides/configuration-advanced/resharding.md b/content/en/docs/user-guides/configuration-advanced/resharding.md
index c4be04c05..115968538 100644
--- a/content/en/docs/user-guides/configuration-advanced/resharding.md
+++ b/content/en/docs/user-guides/configuration-advanced/resharding.md
@@ -1,10 +1,11 @@
---
title: Resharding
-weight: 1
+weight: 4
+aliases: ['/docs/user-guides/resharding/']
---
{{< info >}}
-This guide follows on from the Get Started guides. Please make sure that you have an [Operator](../../../get-started/operator), [local](../../../get-started/local) or [Helm](../../../get-started/helm) installation ready.
+This guide follows on from the Get Started guides. Please make sure that you have an [Operator](../../../get-started/operator), [local](../../../get-started/local) or [Helm](../../../get-started/helm) installation ready. It also assumes that the [MoveTables](../../migration/move-tables/) user guide has been followed.
{{< /info >}}
## Preparation
@@ -135,8 +136,8 @@ helm upgrade vitess ../../helm/vitess/ -f 301_customer_sharded.yaml
```bash
vtctlclient ApplySchema -sql="$(cat create_commerce_seq.sql)" commerce
vtctlclient ApplyVSchema -vschema="$(cat vschema_commerce_seq.json)" commerce
-vtctlclient ApplySchema -sql="$(cat create_customer_sharded.sql)" customer
vtctlclient ApplyVSchema -vschema="$(cat vschema_customer_sharded.json)" customer
+vtctlclient ApplySchema -sql="$(cat create_customer_sharded.sql)" customer
```
### Using a Local Deployment
@@ -144,8 +145,8 @@ vtctlclient ApplyVSchema -vschema="$(cat vschema_customer_sharded.json)" custome
``` sh
vtctlclient ApplySchema -sql-file create_commerce_seq.sql commerce
vtctlclient ApplyVSchema -vschema_file vschema_commerce_seq.json commerce
-vtctlclient ApplySchema -sql-file create_customer_sharded.sql customer
vtctlclient ApplyVSchema -vschema_file vschema_customer_sharded.json customer
+vtctlclient ApplySchema -sql-file create_customer_sharded.sql customer
```
## Create new shards
diff --git a/content/en/docs/user-guides/configuration-advanced/tracing.md b/content/en/docs/user-guides/configuration-advanced/tracing.md
index 5d5f88828..58552f251 100644
--- a/content/en/docs/user-guides/configuration-advanced/tracing.md
+++ b/content/en/docs/user-guides/configuration-advanced/tracing.md
@@ -1,6 +1,7 @@
---
title: Tracing
weight: 14
+aliases: ['/docs/user-guides/tracing/']
---
# Vitess tracing
diff --git a/content/en/docs/user-guides/configuration-advanced/unmanaged-tablet.md b/content/en/docs/user-guides/configuration-advanced/unmanaged-tablet.md
index 22e5c1bc4..c850aec10 100644
--- a/content/en/docs/user-guides/configuration-advanced/unmanaged-tablet.md
+++ b/content/en/docs/user-guides/configuration-advanced/unmanaged-tablet.md
@@ -1,13 +1,14 @@
---
title: Unmanaged Tablet
weight: 15
+aliases: ['/docs/user-guides/unmanaged-tablet/']
---
{{< info >}}
This guide follows on from the [local](../../../get-started/local) installation guide.
{{< /info >}}
-This guide uses the Vitess components vtctld, Topology Service and VTGate which have already been started in the local installation guide. It assumes that you have an existing MySQL Server setup that you would like to add to Vitess as a new keyspace, which we will call `legacy`. The same set of steps can be used to create a tablet that uses Amazon RDS, Aurora, or Google CloudSQL.
+This guide uses the Vitess components vtctld, Topology Service and VTGate which have already been started in the local installation guide. It assumes that you have an existing MySQL Server setup that you would like to add to Vitess as a new keyspace, which we will call `legacy`. The same set of steps can be used to create a tablet that uses Amazon RDS, AWS Aurora, or Google CloudSQL.
## Ensure all components are up
@@ -82,6 +83,9 @@ vttablet \
-init_populate_metadata &
```
+Note that if your tablet is using a MySQL instance type where you do not have `SUPER` privileges to the database
+(e.g. AWS RDS, AWS Aurora or Google CloudSQL), you should omit the `-init_populate_metadata` flag. The `-init_populate_metadata` flag should only be enabled if the cluster is being managed through Vitess.
+
You should be able to see debug information written to screen confirming Vitess can reach the unmanaged server. A common problem is that you may need to change the authentication plugin to `mysql_native_password` (MySQL 8.0).
Assuming that there are no errors, after a few seconds you can mark the server as externally promoted to master:
diff --git a/content/en/docs/user-guides/configuration-advanced/user-management.md b/content/en/docs/user-guides/configuration-advanced/user-management.md
index 013a5b890..880609bf9 100644
--- a/content/en/docs/user-guides/configuration-advanced/user-management.md
+++ b/content/en/docs/user-guides/configuration-advanced/user-management.md
@@ -1,6 +1,7 @@
---
title: User Management and Authentication
-weight: 5
+weight: 1
+aliases: ['/docs/user-guides/user-management/']
---
Vitess uses its own mechanism for managing users and their permissions through
diff --git a/content/en/docs/user-guides/configuration-basic/_index.md b/content/en/docs/user-guides/configuration-basic/_index.md
index 12443759a..8faa6e968 100644
--- a/content/en/docs/user-guides/configuration-basic/_index.md
+++ b/content/en/docs/user-guides/configuration-basic/_index.md
@@ -1,5 +1,5 @@
---
title: Configuration
description: User guides covering basic configuration concepts
-weight: 1
----
\ No newline at end of file
+weight: 2
+---
diff --git a/content/en/docs/user-guides/configuration-basic/configuring-components.md b/content/en/docs/user-guides/configuration-basic/configuring-components.md
index e85c64110..53e2edaf3 100644
--- a/content/en/docs/user-guides/configuration-basic/configuring-components.md
+++ b/content/en/docs/user-guides/configuration-basic/configuring-components.md
@@ -1,7 +1,7 @@
---
title: Configuring Components
weight: 2
-aliases: ['/docs/launching/server-configuration/', '/docs/user-guides/server-configuration/']
+aliases: ['/docs/launching/server-configuration/', '/docs/user-guides/server-configuration/', '/docs/user-guides/configuring-components/']
---
## Managed MySQL
@@ -359,7 +359,7 @@ This URL prints out a simple "ok" or “not ok” string that can be used to che
#### /querylogz, /debug/querylog, /txlogz, /debug/txlog
-* /debug/querylog is a never-ending stream of currently executing queries with verbose information about each query. This URL can generate a lot of data because it streams every query processed by VTTablet. The details are as per this function: https://github.com/vitessio/vitess/blob/master/go/vt/tabletserver/logstats.go#L202
+* /debug/querylog is a never-ending stream of currently executing queries with verbose information about each query. This URL can generate a lot of data because it streams every query processed by VTTablet. The details are as per this function: https://github.com/vitessio/vitess/blob/master/go/vt/vttablet/tabletserver/tabletenv/logstats.go#L202
* /querylogz is a limited human readable version of /debug/querylog. It prints the next 300 queries by default. The limit can be specified with a limit=N parameter on the URL.
* /txlogz is like /querylogz, but for transactions.
* /debug/txlog is the JSON counterpart to /txlogz.
diff --git a/content/en/docs/user-guides/configuration-basic/exporting-data.md b/content/en/docs/user-guides/configuration-basic/exporting-data.md
index c78dbfc60..7f4439129 100644
--- a/content/en/docs/user-guides/configuration-basic/exporting-data.md
+++ b/content/en/docs/user-guides/configuration-basic/exporting-data.md
@@ -1,6 +1,7 @@
---
title: Exporting data from Vitess
weight: 8
+aliases: ['/docs/user-guides/exporting-data/']
---
Since [VTGate](../../../concepts/vtgate/) supports the MySQL protocol, in many
diff --git a/content/en/docs/user-guides/configuration-basic/production-planning.md b/content/en/docs/user-guides/configuration-basic/production-planning.md
index 68e4c256e..e0669107b 100644
--- a/content/en/docs/user-guides/configuration-basic/production-planning.md
+++ b/content/en/docs/user-guides/configuration-basic/production-planning.md
@@ -1,7 +1,7 @@
---
title: Production Planning
weight: 1
-aliases: ['/docs/launching/production-planning/','/docs/launching/']
+aliases: ['/docs/launching/production-planning/','/docs/launching/', '/docs/user-guides/production-planning']
---
## Provisioning
diff --git a/content/en/docs/user-guides/historical/_index.md b/content/en/docs/user-guides/historical/_index.md
index df2023a1f..783f93059 100644
--- a/content/en/docs/user-guides/historical/_index.md
+++ b/content/en/docs/user-guides/historical/_index.md
@@ -1,5 +1,5 @@
---
-title: Historical
+title: Legacy
description: User guides for features in older version of Vitess
weight: 6
---
\ No newline at end of file
diff --git a/content/en/docs/user-guides/historical/horizontal-sharding.md b/content/en/docs/user-guides/historical/horizontal-sharding.md
index 5da3202de..411d44c04 100644
--- a/content/en/docs/user-guides/historical/horizontal-sharding.md
+++ b/content/en/docs/user-guides/historical/horizontal-sharding.md
@@ -1,6 +1,7 @@
---
title: Horizontal Sharding
weight: 1
+aliases: ['/docs/user-guides/horizontal-sharding/']
---
{{< warning >}}
diff --git a/content/en/docs/user-guides/historical/vertical-split.md b/content/en/docs/user-guides/historical/vertical-split.md
index ce29936f0..f0f9f5060 100644
--- a/content/en/docs/user-guides/historical/vertical-split.md
+++ b/content/en/docs/user-guides/historical/vertical-split.md
@@ -1,10 +1,11 @@
---
title: Vertical Split
weight: 2
+aliases: ['/docs/user-guides/vertical-split/']
---
{{< warning >}}
-In Vitess 6, Vertical Split became obsolete with the introduction of MoveTables! It is recommended to skip this guide, and continue on with the [MoveTables user guide](../../migration/move-tables) instead.
+In Vitess 6, Vertical Split became obsolete with the introduction of MoveTables! It is recommended to skip this guide, and continue on with the [MoveTables user guide](../../migration/move-tables) instead. If you continue please note that all scripts referenced are now contained in a different repo called ['legacy_local'](https://github.com/vitessio/vitess/tree/master/examples/legacy_local).
{{< /warning >}}
{{< info >}}
diff --git a/content/en/docs/user-guides/migration/_index.md b/content/en/docs/user-guides/migration/_index.md
index fb917bbe3..faf9ba602 100644
--- a/content/en/docs/user-guides/migration/_index.md
+++ b/content/en/docs/user-guides/migration/_index.md
@@ -1,5 +1,5 @@
---
title: Migration
description: User guides covering migration to Vitess
-weight: 2
----
\ No newline at end of file
+weight: 3
+---
diff --git a/content/en/docs/user-guides/migration/materialize.md b/content/en/docs/user-guides/migration/materialize.md
index ed4693a56..df050e8d7 100644
--- a/content/en/docs/user-guides/migration/materialize.md
+++ b/content/en/docs/user-guides/migration/materialize.md
@@ -1,6 +1,7 @@
---
title: Materialize
weight: 3
+aliases: ['/docs/user-guides/materialize/']
---
{{< info >}}
@@ -17,18 +18,21 @@ In our example, we will be using `Materialize` to perform something similar to t
Let's start by simulating this situation by loading sample data:
```sql
-mysql < ../common/insert_commerce_data.sql
+# On helm and local installs:
+mysql < /usr/local/vitess/examples/common/insert_commerce_data.sql
+# With operator:
+mysql --table < insert_commerce_data.sql
```
We can look at what we just inserted:
```sh
# On helm and local installs:
-mysql --table < ../common/select_commerce_data.sql
+mysql --table < /usr/local/vitess/examples/common/select_commerce_data.sql
# With operator:
mysql --table < select_commerce_data.sql
-Using commerce/0
+Using commerce
Customer
+-------------+--------------------+
| customer_id | email |
@@ -58,7 +62,7 @@ COrder
+----------+-------------+----------+-------+
```
-Note that we are using keyspace `commerce/0` to select data from our tables.
+Note that we are using keyspace `commerce` to select data from our tables.
## Planning to use Materialize
diff --git a/content/en/docs/user-guides/migration/migrate-data.md b/content/en/docs/user-guides/migration/migrate-data.md
index ceb436372..45da1de8d 100644
--- a/content/en/docs/user-guides/migration/migrate-data.md
+++ b/content/en/docs/user-guides/migration/migrate-data.md
@@ -1,6 +1,7 @@
---
title: Migrating data into Vitess
weight: 1
+aliases: ['/docs/user-guides/migrate-data/']
---
# Introduction
diff --git a/content/en/docs/user-guides/migration/move-tables.md b/content/en/docs/user-guides/migration/move-tables.md
index a0cf93f8d..801a4e88c 100644
--- a/content/en/docs/user-guides/migration/move-tables.md
+++ b/content/en/docs/user-guides/migration/move-tables.md
@@ -1,6 +1,7 @@
---
title: MoveTables
weight: 2
+aliases: ['/docs/user-guides/move-tables/']
---
{{< info >}}
@@ -15,15 +16,18 @@ As a stepping stone towards splitting a single table across multiple servers (sh
Let's start by simulating this situation by loading sample data:
-```sql
-mysql < ../common/insert_commerce_data.sql
+```sh
+# On helm and local installs:
+mysql < /usr/local/vitess/examples/common/insert_commerce_data.sql
+# With operator:
+mysql --table < insert_commerce_data.sql
```
We can look at what we just inserted:
```sh
# On helm and local installs:
-mysql --table < ../common/select_commerce_data.sql
+mysql --table < /usr/local/vitess/examples/common/select_commerce_data.sql
# With operator:
mysql --table < select_commerce_data.sql
@@ -66,7 +70,7 @@ In this scenario, we are going to add the `customer` keyspace to the `commerce`
## Show our current tablets
```sh
-$ echo "show vitess_tablets;" | mysql --table
+$ mysql --table --execute="show vitess_tablets"
+-------+----------+-------+------------+---------+------------------+-----------+----------------------+
| Cell | Keyspace | Shard | TabletType | State | Alias | Hostname | MasterTermStartTime |
+-------+----------+-------+------------+---------+------------------+-----------+----------------------+
@@ -158,7 +162,7 @@ vtctlclient ReloadSchemaKeyspace customer
## Show our old and new tablets
```sh
-$ echo "show vitess_tablets;" | mysql --table
+$ mysql --table --execute="show vitess_tablets"
+-------+----------+-------+------------+---------+------------------+-----------+----------------------+
| Cell | Keyspace | Shard | TabletType | State | Alias | Hostname | MasterTermStartTime |
+-------+----------+-------+------------+---------+------------------+-----------+----------------------+
diff --git a/content/en/docs/user-guides/operating-vitess/_index.md b/content/en/docs/user-guides/operating-vitess/_index.md
index 9b481b632..9b91e1356 100644
--- a/content/en/docs/user-guides/operating-vitess/_index.md
+++ b/content/en/docs/user-guides/operating-vitess/_index.md
@@ -2,5 +2,5 @@
title: Operational
description: User guides for covering operational aspects of Vitess
description: User guides covering operational aspects of Vitess
-weight: 4
----
\ No newline at end of file
+weight: 5
+---
diff --git a/content/en/docs/user-guides/operating-vitess/backup-and-restore.md b/content/en/docs/user-guides/operating-vitess/backup-and-restore.md
index 2f8ea9df2..dff7eddfb 100644
--- a/content/en/docs/user-guides/operating-vitess/backup-and-restore.md
+++ b/content/en/docs/user-guides/operating-vitess/backup-and-restore.md
@@ -1,7 +1,7 @@
---
title: Backup and Restore
weight: 2
-aliases: ['/user-guide/backup-and-restore.html']
+aliases: ['/docs/user-guides/backup-and-restore/']
---
Backup and Restore are integrated features provided by tablets managed by Vitess. As well as using _backups_ for data integrity, Vitess will also create and restore backups for provisioning new tablets in an existing shard.
diff --git a/content/en/docs/user-guides/operating-vitess/making-schema-changes.md b/content/en/docs/user-guides/operating-vitess/making-schema-changes.md
deleted file mode 100644
index 3113a2102..000000000
--- a/content/en/docs/user-guides/operating-vitess/making-schema-changes.md
+++ /dev/null
@@ -1,35 +0,0 @@
----
-title: Making Schema Changes
-weight: 3
-aliases: ['/docs/schema-management/mysql-schema/', '/docs/user-guides/mysql-schema/']
----
-
-For applying schema changes for MySQL instances managed by Vitess, there are a few options.
-
-## ApplySchema
-
-`ApplySchema` is a `vtctlclient` command that can be used to apply a schema to a keyspace. The main advantage of using this
-tool is that it performs some sanity checks about the schema before applying it. For example, if the schema change
-affects too many rows of a table, it will reject it.
-
-However, the downside is that it is a little too strict, and may not work for all use cases.
-
-## VTGate
-
-You can send a DDL statement directly to a VTGate just like you would send to a MySQL instance. If the target is a sharded keyspace,
-then the DDL would be scattered to all shards.
-
-If a specific shard fails you can target it directly using the `keyspace/shard` syntax to retry the apply just to that shard.
-
-If VTGate does not recognize a DDL syntax, the statement will get rejected.
-
-This approach is not recommended for changing large tables.
-
-## Directly to MySQL
-
-You can apply schema changes directly to the underlying MySQL shard master instances. VTTablet will eventually notice the change
-and update itself (this is controlled by the `-queryserver-config-schema-reload-time` parameter and defaults to 1800 seconds).
-You can also explicitly issue the `vtctlclient` `ReloadSchema` command to make it reload immediately.
-
-This approach can be extended to use schema deployment tools like `gh-ost` or `pt-online-schema-change`. Using these schema
-deployment tools is the recommended approach for large tables, because they incur no downtime.
diff --git a/content/en/docs/user-guides/operating-vitess/upgrading-vitess.md b/content/en/docs/user-guides/operating-vitess/upgrading-vitess.md
index 10c9b0d5f..9aefce837 100644
--- a/content/en/docs/user-guides/operating-vitess/upgrading-vitess.md
+++ b/content/en/docs/user-guides/operating-vitess/upgrading-vitess.md
@@ -1,7 +1,7 @@
---
title: Upgrading Vitess
weight: 1
-aliases: ['/docs/user-guides/upgrading/']
+aliases: ['/docs/user-guides/upgrading/', '/docs/user-guides/upgrading-vitess/']
---
This document highlights things to be aware of when upgrading a Vitess production installation to a newer Vitess release.
diff --git a/content/en/docs/user-guides/schema-changes/_index.md b/content/en/docs/user-guides/schema-changes/_index.md
index 2c119ee94..8aafbe888 100644
--- a/content/en/docs/user-guides/schema-changes/_index.md
+++ b/content/en/docs/user-guides/schema-changes/_index.md
@@ -2,7 +2,7 @@
title: Making Schema Changes
description:
weight: 11
-aliases: ['/docs/schema-management/schema-changes/', '/docs/user-guides/schema-changes/']
+aliases: ['/docs/schema-management/mysql-schema/', '/docs/user-guides/mysql-schema/', '/docs/user-guides/making-schema-changes/', '/docs/schema-management/schema-changes/', '/docs/user-guides/schema-changes/']
---
This user guide describes the problem space of schema changes and the various approaches you may use with Vitess.
diff --git a/content/en/docs/user-guides/schema-changes/managed-online-schema-changes.md b/content/en/docs/user-guides/schema-changes/managed-online-schema-changes.md
index e8535ddf3..947bb901e 100644
--- a/content/en/docs/user-guides/schema-changes/managed-online-schema-changes.md
+++ b/content/en/docs/user-guides/schema-changes/managed-online-schema-changes.md
@@ -4,20 +4,28 @@ weight: 2
aliases: ['/docs/user-guides/managed-online-schema-changes/']
---
-**Note:** this feature is **EXPERIMENTAL**. Also, the syntax for online-DDL is **subject to change**.
+**Note:** this feature is **EXPERIMENTAL**.
-Vitess offers managed, online schema migrations, via [gh-ost](https://github.com/github/gh-ost) and [pt-online-schema-change](https://www.percona.com/doc/percona-toolkit/3.0/pt-online-schema-change.html). As a quick breakdown:
+Vitess offers managed, online schema migrations, via [gh-ost](https://github.com/github/gh-ost) and [pt-online-schema-change](https://www.percona.com/doc/percona-toolkit/3.0/pt-online-schema-change.html) for `ALTER TABLE` statements, and via [table lifecycle](../../../reference/features/table-lifecycle/) for `DROP TABLE` statements. As a quick breakdown:
-- Vitess recognizes a special `ALTER TABLE` syntax that indicates an online schema change request.
-- Vitess responds to an online schema change request with a job ID
-- Vitess resolves affected shards
-- A shard's `primary` tablet schedules the migration to run when possible
-- The tablets run migrations via `gh-ost` or `pt-online-schema-change`
-- Vitess provides the user a mechanism to view migration status, cancel or retry migrations, based on the job ID
+- Using the standard MySQL `CREATE TABLE`, `ALTER TABLE` and `DROP TABLE` syntax.
+- With either `@@ddl_strategy` session variable set, or `-ddl_strategy` command line flag.
+- Vitess responds to an online schema change request with a job ID.
+- Vitess resolves affected shards.
+- A shard's `primary` tablet schedules the migration to run when possible.
+- Tablets will independently run schema migrations:
+ - `ALTER TABLE` statements run via `gh-ost` or `pt-online-schema-change`.
+ - `CREATE TABLE` statements run directly.
+ - `DROP TABLE` statements run [safely and lazily](../../../reference/features/table-lifecycle/).
+- Vitess provides the user a mechanism to view migration status, cancel or retry migrations, based on the job ID.
## Syntax
-**Note:** while this feature is experimental the syntax is subject to change.
+**Note:** the syntax is subject to change while this feature is in _experimental_ state.
+
+### ALTER TABLE
+
+Use the standard MySQL `ALTER TABLE` syntax to run online DDL. Whether your schema migration runs synchronously (the default MySQL behavior) or asynchronously (aka online), is determined by the value of command line flags or a session variable.
We assume we have a keyspace (schema) called `commerce`, with a table called `demo`, that has the following definition:
@@ -29,29 +37,57 @@ CREATE TABLE `demo` (
) ENGINE=InnoDB
```
-The following syntax is valid and is interpreted by Vitess as an online schema change request:
-
+Consider the following schema migration statement:
```sql
-ALTER WITH 'gh-ost' TABLE demo modify id bigint unsigned;
-ALTER WITH 'gh-ost' '--max-load="Threads_running=200"' TABLE demo modify id bigint unsigned;
-ALTER WITH 'pt-osc' TABLE demo ADD COLUMN created_timestamp TIMESTAMP NOT NULL;
-ALTER WITH 'pt-osc' '--null-to-not-null' TABLE demo ADD COLUMN created_timestamp TIMESTAMP NOT NULL;
+ALTER TABLE demo MODIFY id bigint UNSIGNED;
```
-`gh-ost` and `pt-osc` are the only supported values. Any other value is a syntax error. Specifics about `gh-ost` and `pt-online-schema-change` follow later on.
+This statement can be executed as:
+
+- a `gh-ost`, managed online migration
+- a `pt-online-schema-change`, managed online migration
+- a synchronous, [unmanaged schema change](../unmanaged-schema-changes/)
+
+`gh-ost` and `pt-online-schema-change` (abbreviated as `pt-osc`) are the supported online DDL methods. Specifics about `gh-ost` and `pt-online-schema-change` follow later on.
+
+How the migration is executed depends on your `ddl_strategy` configuration, and how you run your query: via `vtctl/vtctlclient` or `VTGate`. Details follow.
+
+### CREATE TABLE
+
+Use the standard MySQL `CREATE TABLE` syntax. The query goes through the same [migration flow](#migration-flow-and-states) as `ALTER TABLE` does. The tablets eventually run the query directly on the MySQL backends.
+
+### DROP TABLE
+
+Use the standard MySQL `DROP TABLE` syntax. The query goes through the same [migration flow](#migration-flow-and-states) as `ALTER TABLE` does. Tables are not immediately dropped. Instead, they are renamed into special names, recognizable by the table lifecycle mechanism, to then slowly and safely transition through lifecycle states into finally getting dropped.
-You may use this syntax either with `vtctlclient` or via `vtgate`
+### Statement transformations
+
+Vitess may modify your queries to qualify for online DDL statement. Modifications include:
+
+- A multi-table `DROP` statement is replaced by multiple `DROP` statements, each operating on a single table (and each tracked by its own job ID).
+- A `CREATE INDEX` statement is replaced by the equivalent `ALTER TABLE` statement.
+
+## ddl_strategy
+
+You will set either `@@ddl_strategy` session variable, or `-ddl_strategy` command line flag, to control your schema migration strategy, and specifically, to enable and configure online DDL. Details follow in next sections. Some initial examples:
+
+- The value `"direct"`, meaning not an online DDL. The empty value (`""`) is also interpreted as `direct`.
+- The value `"gh-ost"` instructs Vitess to run an `ALTER TABLE` online DDL via `gh-ost`.
+- The value `"pt-osc"` instructs Vitess to run an `ALTER TABLE` online DDL via `pt-online-schema-change`.
+- You may specify arguments for your tool of choice, e.g. `"gh-ost --max-load Threads_running=200"`. Details follow.
## ApplySchema
-Invocation is similar to direct DDL statements. However, the response is different:
+You may use `vtctl` or `vtctlclient` (the two are interchangeable for the purpose of this document) to apply schema changes. The `ApplySchema` command supports both synchronous and online schema migrations. To run an online schema migration you will supply the `-ddl_strategy` command line flag:
```shell
-$ vtctlclient ApplySchema -sql "ALTER WITH 'gh-ost' TABLE demo modify id bigint unsigned" commerce
+$ vtctlclient ApplySchema -ddl_strategy "gh-ost" -sql "ALTER TABLE demo MODIFY id bigint UNSIGNED" commerce
a2994c92_f1d4_11ea_afa3_f875a4d24e90
```
-When the user indicates online schema change (aka online DDL), `vtctl` registers an online-DDL request with global `topo`. This generates a job ID for tracking. `vtctl` does not try to resolve the shards nor the `primary` tablets. The command returns immediately, without waiting for the migration(s) to start. It prints out the job ID (`a2994c92_f1d4_11ea_afa3_f875a4d24e90` in the above)
+Notice how `ApplySchema` responds to an online DDL request with a UUID output. When the user indicates online schema change (aka online DDL), `vtctl` registers an online-DDL request with global `topo`. This generates a job ID for tracking. `vtctl` does not try to resolve the shards nor the `primary` tablets. The command returns immediately, without waiting for the migration(s) to start.
+
+The command prints a job ID for each of the DDL statements. In the above we issue a single DDL statemnt, and `vtctl` responds with a single job ID (`a2994c92_f1d4_11ea_afa3_f875a4d24e90`)
If we immediately run `SHOW CREATE TABLE`, we are likely to still see the old schema:
```sql
@@ -64,18 +100,18 @@ CREATE TABLE `demo` (
) ENGINE=InnoDB
```
-We discuss how the migration jobs get scheduled and executed shortly. We will use the job ID for tracking.
+That's because the migration is to be processed and executed in the future. We discuss how the migration jobs get scheduled and executed shortly. We will use the job ID for tracking.
-`ApplySchema` will have vitess run some validations:
+`ApplySchema` will have Vitess run some validations:
```shell
-$ vtctlclient ApplySchema -sql "ALTER WITH 'gh-ost' TABLE demo add column status int" commerce
+$ vtctlclient ApplySchema -ddl_strategy "gh-ost" -sql "ALTER TABLE demo add column status int" commerce
E0908 16:17:07.651284 3739130 main.go:67] remote error: rpc error: code = Unknown desc = schema change failed, ExecuteResult: {
"FailedShards": null,
"SuccessShards": null,
"CurSQLIndex": 0,
"Sqls": [
- "ALTER WITH 'gh-ost' TABLE demo add column status int"
+ "ALTER TABLE demo add column status int"
],
"ExecutorErr": "rpc error: code = Unknown desc = TabletManager.PreflightSchema on zone1-0000000100 error: /usr/bin/mysql: exit status 1, output: ERROR 1060 (42S21) at line 3: Duplicate column name 'status'\n: /usr/bin/mysql: exit status 1, output: ERROR 1060 (42S21) at line 3: Duplicate column name 'status'\n",
"TotalTimeSpent": 144283260
@@ -86,13 +122,21 @@ Vitess was able to determine that the migration is invalid because a column name
## VTGate
-You may run online DDL directly from VTGate. For example:
+When connected to `VTGate`, your schema migration strategy is determined by:
+
+- `-ddl_strategy` command line flag, or:
+- `@@ddl_strategy` session variable.
+
+As a simple example, you may control the execution of a migration as follows:
```shell
$ mysql -h 127.0.0.1 -P 15306 commerce
Welcome to the MySQL monitor. Commands end with ; or \g.
-mysql> ALTER WITH 'pt-osc' TABLE demo ADD COLUMN sample INT;
+mysql> SET @@ddl_strategy='pt-osc';
+Query OK, 0 rows affected (0.00 sec)
+
+mysql> ALTER TABLE demo ADD COLUMN sample INT;
+--------------------------------------+
| uuid |
+--------------------------------------+
@@ -101,8 +145,15 @@ mysql> ALTER WITH 'pt-osc' TABLE demo ADD COLUMN sample INT;
1 row in set (0.00 sec)
```
-Just like in the previous example, `VTGate` identifies that this is an online schema change request, and persists it in global `topo`, returning a job ID for tracking. Migration does not start immediately.
+Just like in the `ApplySchema` example, `VTGate` identifies that this is an online schema change request, and persists it in global `topo`, returning a job ID for tracking. Migration does not start immediately.
+
+`@@ddl_strategy` behaves like a MySQL session variable, though is only recognized by `VTGate`. Setting `@@ddl_strategy` only applies to that same connection and does not affect other connections. You may subsequently set `@@ddl_strategy` to different value.
+The initial value for `@@ddl_strategy` is derived from the `vtgate -ddl_strategy` command line flag. Examples:
+
+- If you run `vtgate` without `-ddl_strategy`, then `@@ddl_strategy` defaults to `'direct'`, which implies schema migrations are synchronous. You will need to `set @@ddl_strategy='gh-ost'` to run followup `ALTER TABLE` statements via `gh-ost`.
+- If you run `vtgate -ddl_strategy "gh-ost"`, then `@@ddl_strategy` defaults to `'gh-ost'` in each new session. Any `ALTER TABLE` will run via `gh-ost`. You may `set @@ddl_strategy='pt-osc'` to make migrations run through `pt-online-schema-change`, or `set @@ddl_strategy='direct'` to run migrations synchronously.
+
## Migration flow and states
We highlight how Vitess manages migrations internally, and explain what states a migration goes through.
@@ -116,12 +167,20 @@ We highlight how Vitess manages migrations internally, and explain what states a
- A shard's `primary` tablet owns running the migration. It is independent of other shards. It will schedule the migration to run when possible. A tablet will not run two migrations at the same time.
- A migration is first created in `queued` state.
- If the tablet sees queued migration, and assuming there's no reason to wait, it picks the oldest requested migration in `queued` state, and moves it to `ready` state.
-- Tablet then prepares for the migration. It creates a MySQL account with a random password, to be used by this migration only. It creates the command line invocation, and extra scripts if possible.
-- The tablet then runs the migration. Whether `gh-ost` or `pt-online-schema-change`, it first runs in _dry run_ mode, and, if successful, in actual _execute_ mode. The migration is then in `running` state.
-- The migration will either run to completion, fail, or be interrupted. If successful, it transitions into `complete` state, which is the end of the road for that migration. If failed or interrupted, it transitions to `failed` state. The user may choose to _retry_ failed migrations (see below).
-- The user is able to _cancel_ a migration (details below). If the migration hasn't started yet, it transitions to `cancelled` state. If the migration is `running`, then it is interrupted, and is expected to transition into `failed` state.
-
-By way of illustration, suppose a migration is now in `running` state, and is expected to keep on running for the next few hours. The user may initiate a new `ALTER WITH 'gh-ost' TABLE...` statement. It will persist in global `topo`. `vtctld` will pick it up and advertise it to the relevant tablets. Each will persist the migration request in `queued` state. None will run the migration yet, since another migration is already in progress. In due time, and when the executing migration completes (whether successfully or not), and assuming no other migrations are `queued`, the `primary` tablets, each in its own good time, will execute the new migration.
+- For `ALTER TABLE` statements:
+ - Tablet prepares for the migration. It creates a MySQL account with a random password, to be used by this migration only. It creates the command line invocation, and extra scripts if possible.
+ - The tablet then runs the migration. Whether `gh-ost` or `pt-online-schema-change`, it first runs in _dry run_ mode, and, if successful, in actual _execute_ mode. The migration is then in `running` state.
+ - The migration will either run to completion, fail, or be interrupted. If successful, it transitions into `complete` state, which is the end of the road for that migration. If failed or interrupted, it transitions to `failed` state. The user may choose to _retry_ failed migrations (see below).
+ - The user is able to _cancel_ a migration (details below). If the migration hasn't started yet, it transitions to `cancelled` state. If the migration is `running`, then it is interrupted, and is expected to transition into `failed` state.
+- For `CREATE TABLE` statements:
+ - Tablet runs the statement directly on the MySQL backend
+- For `DROP TABLE` statements:
+ - A multi-table `DROP TABLE` statement is converted to multiple single-table `DROP TABLE` statements
+ - Each `DROP TABLE` is internally replaced with a `RENAME TABLE`
+ - Table is renamed using a special naming convention, identified by the [Table Lifecycle](../../../reference/features/table-lifecycle/) mechanism
+ - As result, a single `DROP TABLE` statement may generate multiple distinct migrations with distinct migration UUIDs.
+
+By way of illustration, suppose a migration is now in `running` state, and is expected to keep on running for the next few hours. The user may initiate a new `ALTER TABLE...` statement. It will persist in global `topo`. `vtctld` will pick it up and advertise it to the relevant tablets. Each will persist the migration request in `queued` state. None will run the migration yet, since another migration is already in progress. In due time, and when the executing migration completes (whether successfully or not), and assuming no other migrations are `queued`, the `primary` tablets, each in its own good time, will execute the new migration.
At this time, the user is responsible to track the state of all migrations. VTTablet does not report back to `vtctld`. This may change in the future.
@@ -133,62 +192,62 @@ At this time, there are no automated retries. For example, a failover on a shard
You may track the status of a single migration, of all or recent migrations, or of migrations in a specific state. Examples:
```shell
-$ vtctlclient OnlineDDL commerce show ab3ffdd5_f25c_11ea_bab4_0242c0a8b007
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| test-0000000201 | 40-80 | vt_commerce | demo | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:24:33 | 2020-09-09 05:24:34 | complete |
-| test-0000000301 | 80-c0 | vt_commerce | demo | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
-| test-0000000401 | c0- | vt_commerce | demo | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
-| test-0000000101 | -40 | vt_commerce | demo | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+-$ vtctlclient OnlineDDL commerce show ab3ffdd5_f25c_11ea_bab4_0242c0a8b007
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | ddl_action | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:24:33 | 2020-09-09 05:24:34 | complete |
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
+| test-0000000401 | c0- | vt_commerce | demo | alter | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
+| test-0000000101 | -40 | vt_commerce | demo | alter | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
$ vtctlclient OnlineDDL commerce show 8a797518_f25c_11ea_bab4_0242c0a8b007
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| test-0000000401 | c0- | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | running |
-| test-0000000201 | 40-80 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | 2020-09-09 05:23:33 | complete |
-| test-0000000301 | 80-c0 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | running |
-| test-0000000101 | -40 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | running |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | ddl_action | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| test-0000000401 | c0- | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | running |
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | 2020-09-09 05:23:33 | complete |
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | running |
+| test-0000000101 | -40 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | running |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
$ vtctlclient OnlineDDL commerce show 8a797518_f25c_11ea_bab4_0242c0a8b007
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| test-0000000401 | c0- | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
-| test-0000000101 | -40 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
-| test-0000000301 | 80-c0 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
-| test-0000000201 | 40-80 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | 2020-09-09 05:23:33 | complete |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | ddl_action | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| test-0000000401 | c0- | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
+| test-0000000101 | -40 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | 2020-09-09 05:23:33 | complete |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
$ vtctlclient OnlineDDL commerce show recent
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| test-0000000201 | 40-80 | vt_commerce | demo | 63b5db0c_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:22:41 | 2020-09-09 05:22:42 | complete |
-| test-0000000201 | 40-80 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | 2020-09-09 05:23:33 | complete |
-| test-0000000201 | 40-80 | vt_commerce | demo | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:24:33 | 2020-09-09 05:24:34 | complete |
-| test-0000000301 | 80-c0 | vt_commerce | demo | 63b5db0c_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:22:41 | 2020-09-09 05:22:42 | complete |
-| test-0000000301 | 80-c0 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
-| test-0000000301 | 80-c0 | vt_commerce | demo | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
-| test-0000000401 | c0- | vt_commerce | demo | 63b5db0c_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:22:41 | 2020-09-09 05:22:42 | complete |
-| test-0000000401 | c0- | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
-| test-0000000401 | c0- | vt_commerce | demo | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
-| test-0000000101 | -40 | vt_commerce | demo | 63b5db0c_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:22:41 | 2020-09-09 05:22:42 | complete |
-| test-0000000101 | -40 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
-| test-0000000101 | -40 | vt_commerce | demo | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | ddl_action | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | 63b5db0c_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:22:41 | 2020-09-09 05:22:42 | complete |
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | 2020-09-09 05:23:33 | complete |
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:24:33 | 2020-09-09 05:24:34 | complete |
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | 63b5db0c_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:22:41 | 2020-09-09 05:22:42 | complete |
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
+| test-0000000401 | c0- | vt_commerce | demo | alter | 63b5db0c_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:22:41 | 2020-09-09 05:22:42 | complete |
+| test-0000000401 | c0- | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
+| test-0000000401 | c0- | vt_commerce | demo | alter | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
+| test-0000000101 | -40 | vt_commerce | demo | alter | 63b5db0c_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:22:41 | 2020-09-09 05:22:42 | complete |
+| test-0000000101 | -40 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
+| test-0000000101 | -40 | vt_commerce | demo | alter | ab3ffdd5_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:25:13 | 2020-09-09 05:25:14 | complete |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
$ vtctlclient OnlineDDL commerce show failed
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| test-0000000301 | 80-c0 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
-| test-0000000401 | c0- | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
-| test-0000000101 | -40 | vt_commerce | demo | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | ddl_action | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
+| test-0000000401 | c0- | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
+| test-0000000101 | -40 | vt_commerce | demo | alter | 8a797518_f25c_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 05:23:32 | | failed |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
```
The syntax for tracking migrations is:
@@ -214,14 +273,14 @@ Example:
```shell
$ vtctlclient OnlineDDL commerce show 2201058f_f266_11ea_bab4_0242c0a8b007
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| test-0000000301 | 80-c0 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | running |
-| test-0000000101 | -40 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | running |
-| test-0000000401 | c0- | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | running |
-| test-0000000201 | 40-80 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | running |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | ddl_action | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | running |
+| test-0000000101 | -40 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | running |
+| test-0000000401 | c0- | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | running |
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | running |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
$ vtctlclient OnlineDDL commerce cancel 2201058f_f266_11ea_bab4_0242c0a8b007
+-----------------+--------------+
@@ -234,35 +293,78 @@ $ vtctlclient OnlineDDL commerce cancel 2201058f_f266_11ea_bab4_0242c0a8b007
+-----------------+--------------+
$ vtctlclient OnlineDDL commerce show 2201058f_f266_11ea_bab4_0242c0a8b007
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| test-0000000401 | c0- | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
-| test-0000000301 | 80-c0 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
-| test-0000000201 | 40-80 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
-| test-0000000101 | -40 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | ddl_action | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| test-0000000401 | c0- | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
+| test-0000000101 | -40 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+```
+
+
+## Cancelling all keyspace migrations
+
+The user may cancel all migrations in a keyspace. A migration is cancellable if it is in `queued`, `ready` or `running` states, as described previously. It is a high impact operation and should be used with care.
+
+The syntax to cancelling all keyspace migrations is:
+
```
+vtctlclient OnlineDDL cancel-all
+```
+
+Example:
+```shell
+$ vtctlclient OnlineDDL commerce show all
++------------------+-------+--------------+-------------+--------------------------------------+----------+-------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++------------------+-------+--------------+-------------+--------------------------------------+----------+-------------------+---------------------+------------------+
+| zone1-0000000100 | 0 | vt_commerce | corder | 2c581994_353a_11eb_8b72_f875a4d24e90 | gh-ost | | | queued |
+| zone1-0000000100 | 0 | vt_commerce | corder | 2c6420c9_353a_11eb_8b72_f875a4d24e90 | gh-ost | | | queued |
+| zone1-0000000100 | 0 | vt_commerce | corder | 2c7040df_353a_11eb_8b72_f875a4d24e90 | gh-ost | | | queued |
+| zone1-0000000100 | 0 | vt_commerce | corder | 2c7c0572_353a_11eb_8b72_f875a4d24e90 | gh-ost | | | queued |
+| zone1-0000000100 | 0 | vt_commerce | corder | 2c87f7cd_353a_11eb_8b72_f875a4d24e90 | gh-ost | | | queued |
++------------------+-------+--------------+-------------+--------------------------------------+----------+-------------------+---------------------+------------------+
+
+$ vtctlclient OnlineDDL commerce cancel-all
++------------------+--------------+
+| Tablet | RowsAffected |
++------------------+--------------+
+| zone1-0000000100 | 5 |
++------------------+--------------+
+
+ vtctlclient OnlineDDL commerce show all
++------------------+-------+--------------+-------------+--------------------------------------+----------+-------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++------------------+-------+--------------+-------------+--------------------------------------+----------+-------------------+---------------------+------------------+
+| zone1-0000000100 | 0 | vt_commerce | corder | 2c581994_353a_11eb_8b72_f875a4d24e90 | gh-ost | | | cancelled |
+| zone1-0000000100 | 0 | vt_commerce | corder | 2c6420c9_353a_11eb_8b72_f875a4d24e90 | gh-ost | | | cancelled |
+| zone1-0000000100 | 0 | vt_commerce | corder | 2c7040df_353a_11eb_8b72_f875a4d24e90 | gh-ost | | | cancelled |
+| zone1-0000000100 | 0 | vt_commerce | corder | 2c7c0572_353a_11eb_8b72_f875a4d24e90 | gh-ost | | | cancelled |
+| zone1-0000000100 | 0 | vt_commerce | corder | 2c87f7cd_353a_11eb_8b72_f875a4d24e90 | gh-ost | | | cancelled |
++------------------+-------+--------------+-------------+--------------------------------------+----------+-------------------+---------------------+------------------+
+```
## Retrying a migration
The user may retry running a migration. If the migration is in `failed` or in `cancelled` state, Vitess will re-run the migration, with exact same arguments as previously intended. If the migration is in any other state, `retry` does nothing.
-It is not possible to retry a migration with different options. e.g. if the user initially runs `ALTER WITH 'gh-ost' '--max-load Threads_running=200' TABLE demo MODIFY id BIGINT` and the migration failed, it is not possible to retry with `'--max-load Threads_running=500'`.
+It is not possible to retry a migration with different options. e.g. if the user initially runs `ALTER TABLE demo MODIFY id BIGINT` with `@@ddl_strategy='gh-ost --max-load Threads_running=200'` and the migration fails, retrying it will use exact same options. It is not possible to retry with `@@ddl_strategy='gh-ost --max-load Threads_running=500'`.
Continuing the above example, where we cancelled a migration while running, we now retry it:
```shell
$ vtctlclient OnlineDDL commerce show 2201058f_f266_11ea_bab4_0242c0a8b007
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| test-0000000401 | c0- | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
-| test-0000000301 | 80-c0 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
-| test-0000000201 | 40-80 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
-| test-0000000101 | -40 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | ddl_action | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| test-0000000401 | c0- | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
+| test-0000000101 | -40 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:32:31 | | failed |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
$ vtctlclient OnlineDDL commerce retry 2201058f_f266_11ea_bab4_0242c0a8b007
+-----------------+--------------+
@@ -275,26 +377,43 @@ $ vtctlclient OnlineDDL commerce retry 2201058f_f266_11ea_bab4_0242c0a8b007
+-----------------+--------------+
$ vtctlclient OnlineDDL commerce show 2201058f_f266_11ea_bab4_0242c0a8b007
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+-------------------+---------------------+------------------+
-| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+-------------------+---------------------+------------------+
-| test-0000000201 | 40-80 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | | | queued |
-| test-0000000101 | -40 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | | | queued |
-| test-0000000301 | 80-c0 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | | | queued |
-| test-0000000401 | c0- | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | | | queued |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+-------------------+---------------------+------------------+
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+-------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | ddl_action | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+-------------------+---------------------+------------------+
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | | | queued |
+| test-0000000101 | -40 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | | | queued |
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | | | queued |
+| test-0000000401 | c0- | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | | | queued |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+-------------------+---------------------+------------------+
$ vtctlclient OnlineDDL commerce show 2201058f_f266_11ea_bab4_0242c0a8b007
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| Tablet | shard | mysql_schema | mysql_table | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
-| test-0000000101 | -40 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:37:33 | | running |
-| test-0000000401 | c0- | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:37:33 | | running |
-| test-0000000201 | 40-80 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:37:33 | | running |
-| test-0000000301 | 80-c0 | vt_commerce | demo | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:37:33 | | running |
-+-----------------+-------+--------------+-------------+--------------------------------------+----------+---------------------+---------------------+------------------+
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| Tablet | shard | mysql_schema | mysql_table | ddl_action | migration_uuid | strategy | started_timestamp | completed_timestamp | migration_status |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
+| test-0000000101 | -40 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:37:33 | | running |
+| test-0000000401 | c0- | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:37:33 | | running |
+| test-0000000201 | 40-80 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:37:33 | | running |
+| test-0000000301 | 80-c0 | vt_commerce | demo | alter | 2201058f_f266_11ea_bab4_0242c0a8b007 | gh-ost | 2020-09-09 06:37:33 | | running |
++-----------------+-------+--------------+-------------+------------+--------------------------------------+----------+---------------------+---------------------+------------------+
```
+## Auto retry after failure
+
+Vitess keeps track of:
+
+- which `vttablet` initiated the migration
+- how many times a migration has been retried
+- whether a migration failed due to a `vttablet` failure (as is the case in a failover scenario)
+
+Vitess will auto-retry a failed migration when:
+
+- The migration failed due to a `vttablet` failure, and
+- it has not been retried (this is a temporary restriction)
+
+The migration will be transitioned into `queued` state, as if the user requested a `retry` operation. Note that this takes place on a per-shard basis.
+
+The primary use case is a primary failure and failover. The newly promoted tablet will be able to retry the migration that broke during the previous primary failure. To clarify, the migration will start anew, as at this time there is no mechanism to resume a broken migration.
+
## gh-ost and pt-online-schema-change
The user must pick one of these migration tools. The tools differ in features, operation, load, and more.
@@ -312,11 +431,11 @@ To be able to run online schema migrations via `gh-ost`:
Vitess automatically creates a MySQL account for the migration, with a randomly generated password. The account is destroyed at the end of the migration.
-Vitess takes care of setting up the necessary command line flags. It automatically creates a hooks directory and populates it with hooks that report `gh-ost`'s progress back to Vitess. You may supply additional flags for your migration as part of the `ALTER` statement. Examples:
+Vitess takes care of setting up the necessary command line flags. It automatically creates a hooks directory and populates it with hooks that report `gh-ost`'s progress back to Vitess. You may supply additional flags for your migration as part of `@@ddl_strategy` session variable (using `VTGate`) or `-ddl_strategy` command line flag (using `vtctl`). Examples:
-- `ALTER WITH 'gh-ost' '--max-load Threads_running=200' TABLE demo MODIFY id BIGINT`
-- `ALTER WITH 'gh-ost' '--critical-load Threads_running=500 --critical-load-hibernate-seconds=60' --default-retries=512 TABLE demo MODIFY id BIGINT`
-- `ALTER WITH 'gh-ost' '--allow-nullable-unique-key --chunk-size 200' TABLE demo MODIFY id BIGINT`
+- `set @@ddl_strategy='gh-ost --max-load Threads_running=200';`
+- `set @@ddl_strategy='gh-ost --max-load Threads_running=200 --critical-load Threads_running=500 --critical-load-hibernate-seconds=60 --default-retries=512';`
+- `vtctl -ddl_strategy "gh-ost --allow-nullable-unique-key --chunk-size 200" ApplySchema ...`
Do not override the following flags: `alter, database, table, execute, max-lag, force-table-names, serve-socket-file, hooks-path, hooks-hint-token, panic-flag-file`.
@@ -333,11 +452,11 @@ Do not override the following flags: `alter, database, table, execute, max-lag,
Vitess automatically creates a MySQL account for the migration, with a randomly generated password. The account is destroyed at the end of the migration.
-Vitess takes care of supplying the command line flags, the DSN, the username & password. It also sets up `PLUGINS` used to communicate migration progress back to the tablet. You may supply additional flags for your migration as part of the `ALTER` statement. Examples:
+Vitess takes care of supplying the command line flags, the DSN, the username & password. It also sets up `PLUGINS` used to communicate migration progress back to the tablet. You may supply additional flags for your migration as part of `@@ddl_strategy` session variable (using `VTGate`) or `-ddl_strategy` command line flag (using `vtctl`). Examples:
-- `ALTER WITH 'pt-osc' '--null-to-not-null' TABLE demo MODIFY id BIGINT`
-- `ALTER WITH 'pt-osc' '--max-load Threads_running=200' TABLE demo MODIFY id BIGINT`
-- `ALTER WITH 'pt-osc' '--alter-foreign-keys-method auto --chunk-size 200' TABLE demo MODIFY id BIGINT`
+- `set @@ddl_strategy='pt-osc --null-to-not-null';`
+- `set @@ddl_strategy='pt-osc --max-load Threads_running=200';`
+- `vtctl -ddl_strategy "pt-osc --alter-foreign-keys-method auto --chunk-size 200" ApplySchema ...`
Vitess tracks the state of the `pt-osc` migration. If it fails, Vitess makes sure to drop the migration triggers. Vitess keeps track of the migration even if the tablet itself restarts for any reason. Normally that would terminate the migration; vitess will cleanup the triggers if so, or will happily let the migration run to completion if not.
@@ -348,7 +467,7 @@ Do not override the following flags: `alter, pid, plugin, dry-run, execute, new-
## Throttling
-Schema migrations use the tablet throttler, which is a cooperative throttler service based on replication lag. The tablet throttler automatically detectes topology `REPLICA` tablets and adapts to changes in the topology. See [Tablet throttler](../../../reference/features/tablet-throttler/).
+`gh-ost` and `pt-online-schema-change` schema migrations use the tablet throttler, which is a cooperative throttler service based on replication lag. The tablet throttler automatically detects topology `REPLICA` tablets and adapts to changes in the topology. See [Tablet throttler](../../../reference/features/tablet-throttler/).
**NOTE** that at this time the tablet throttler is an experimental feature and is opt in. Enable it with `vttablet`'s `-enable-lag-throttler` flag. If the tablet throttler is disabled, schema migrations will not throttle on replication lag.
@@ -371,7 +490,7 @@ For schema migrations, Vitess allows operations on the virtual table `_vt.schema
- `artifacts`: tables created by the migration. This can be used to determine which tables need cleanup.
- `alter`: the exact `alter` statement used by the migration
- `options`: any options passed by the user (e.g. `--max-load=Threads_running=200`)
- - Various timestamps indicating the migratoin progress
+ - Various timestamps indicating the migration progress
Aggregate functions do not work as expected and should be avoided. `LIMIT` and `OFFSET` are not supported.
- `UPDATE`: you may directly update the status of a migration. You may only change status into `cancel` or `retry`, which Vitess interprets similarly to a `vtctlclient OnlineDDL cancel/retry` command. However, you get greater control as you may filter on a specific `shard`.
- `DELETE`: unsupported
diff --git a/content/en/docs/user-guides/schema-changes/unmanaged-schema-changes.md b/content/en/docs/user-guides/schema-changes/unmanaged-schema-changes.md
index 5b2d0ae12..81c3e6dfb 100644
--- a/content/en/docs/user-guides/schema-changes/unmanaged-schema-changes.md
+++ b/content/en/docs/user-guides/schema-changes/unmanaged-schema-changes.md
@@ -1,7 +1,7 @@
---
title: Unmanaged Schema Changes
weight: 3
-aliases: ['/docs/schema-management/unmanaged-schema-changes/', '/docs/user-guides/unmanaged-schema-changes/']
+aliases: ['/docs/user-guides/operating-vitess/making-schema-changes', '/docs/schema-management/unmanaged-schema-changes/', '/docs/user-guides/unmanaged-schema-changes/']
---
Vitess offers multiple approaches to running unmanaged schema changes. Below, we review each of these approaches.
@@ -18,7 +18,7 @@ CREATE TABLE `demo` (
## ApplySchema
-`ApplySchema` is a `vtctlclient` command that can be used to apply a schema change to a keyspace. The main advantage of using this tool is that it performs some sanity checks about the schema before applying it.
+`ApplySchema` is a `vtctlclient` command that can be used to apply a schema change to a keyspace. The main advantage of using this tool is that it performs some sanity checks about the schema before applying it. However, a downside is that it can be a little too strict and may not work for all use cases.
Consider the following examples:
@@ -64,7 +64,7 @@ $ vtctlclient ApplySchema -allow_long_unavailability -sql "ALTER TABLE demo modi
## VTGate
-You may run DDL directly from VTGate. For example:
+You may run DDL directly from VTGate just like you would send to a MySQL instance. For example:
```shell
$ mysql -h 127.0.0.1 -P 15306 commerce
@@ -86,9 +86,9 @@ mysql> ALTER TABLE demo ADD COLUMN sample INT;
Query OK, 0 rows affected (0.04 sec)
```
-In the above we connect to VTGate via the `mysql` command line client, but of course you may connect with any standard MySQL client or from your applicaiton.
+In the above we connect to VTGate via the `mysql` command line client, but of course you may connect with any standard MySQL client or from your application.
-This approach is not recommended for changing large tables.
+Please do note that if VTGate does not recognize a DDL syntax, the statement will get rejected and that this approach is not recommended for changing large tables.
## Directly to MySQL
@@ -102,4 +102,6 @@ You can also explicitly issue the `vtctlclient` `ReloadSchema` command to make i
$ vtctlclient ReloadSchema zone1-0000000100
```
-Users will likely want to deploy schema changes via `gh-ost` or `pt-online-schema-change`, which do not block the table. Vitess offers [managed, online schema changes](../managed-online-schema-changes/) where it automates the invocation and execution of these tools.
+Users will likely want to deploy schema changes via `gh-ost` or `pt-online-schema-change`, which do not block the table. Vitess offers [managed, online schema changes](../managed-online-schema-changes/) where it automates the invocation and execution of these tools. Using these schema
+deployment tools can be a better approach for large tables, because they should incur no downtime.
+
diff --git a/content/en/docs/user-guides/sql/_index.md b/content/en/docs/user-guides/sql/_index.md
index e79cbe08c..67307d95a 100644
--- a/content/en/docs/user-guides/sql/_index.md
+++ b/content/en/docs/user-guides/sql/_index.md
@@ -1,5 +1,5 @@
---
title: SQL Statement Analysis
description: User guides covering analyzing SQL statements
-weight: 3
----
\ No newline at end of file
+weight: 4
+---
diff --git a/content/en/docs/user-guides/sql/vtexplain-in-bulk.md b/content/en/docs/user-guides/sql/vtexplain-in-bulk.md
index e74e39cc4..e6f9cca33 100644
--- a/content/en/docs/user-guides/sql/vtexplain-in-bulk.md
+++ b/content/en/docs/user-guides/sql/vtexplain-in-bulk.md
@@ -1,6 +1,7 @@
---
title: Analyzing SQL statements in bulk
weight: 2
+aliases: ['/docs/user-guides/vtexplain-in-bulk/']
---
# Introduction
diff --git a/content/en/docs/user-guides/sql/vtexplain.md b/content/en/docs/user-guides/sql/vtexplain.md
index 0ca5b93d8..33f48ac3e 100644
--- a/content/en/docs/user-guides/sql/vtexplain.md
+++ b/content/en/docs/user-guides/sql/vtexplain.md
@@ -1,6 +1,7 @@
---
title: Analyzing a SQL statement
weight: 1
+aliases: ['/docs/user-guides/vtexplain/']
---
# Introduction
diff --git a/content/en/docs/user-guides/vschema-guide/_index.md b/content/en/docs/user-guides/vschema-guide/_index.md
new file mode 100644
index 000000000..c60d42ebf
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/_index.md
@@ -0,0 +1,5 @@
+---
+title: VSchema and Query Serving
+description: Configuring VSchema for serving queries
+weight: 1
+---
diff --git a/content/en/docs/user-guides/vschema-guide/advanced-vschema.md b/content/en/docs/user-guides/vschema-guide/advanced-vschema.md
new file mode 100644
index 000000000..1f4bb9df5
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/advanced-vschema.md
@@ -0,0 +1,135 @@
+---
+title: Advanced VSchema Properties
+weight: 11
+---
+
+With the exception of Multi-Column Vindexes, advanced VSchema Properties do not have DDL constructs. They can only be updated through `vtctld` CLI commands.
+
+## Multi-Column Vindexes
+
+Multi-Column Vindexes are useful in the following two use cases:
+
+* Grouping customers by their regions so they can be hosted in specific geographical locations. This may be required for compliance, and also to achieve better performance.
+* For a multi-tenant system, grouping all rows of a tenant in a separate set of shards. This limits the fan out of queries if searching only for rows that are related to a single tenant.
+
+In both cases the leading column is the region or tenant, and is used to form the first few bits of the `keyspace_id`. The second column is used for the bits that follow. Since Vitess shards by keyrange, this approach will naturally group all rows of a region or tenant within the same shard, or within a group of consecutive shards. Since each shard is its own MySQL cluster, these can then be deployed to different regions as needed.
+
+Please refer to [Region-based Sharding](../../configuration-advanced/region-sharding) for an example on how to use the `region_json` vindex.
+
+Currently, the Vindex gets used for assigning a `keyspace_id` at the time of insert and at the time of resharding. Additional vindexes need to be added to the table for routing query constructs that contain WHERE clauses.
+
+Vitess does not have the capability to route a query based on multiple values of a multi-column vindex in a where clause yet. This feature will be added soon.
+
+#### Alternate approach
+
+You have the option to pre-combine the region and id bits into a single column and use that as an input for a single column vindex. This approach achieves the same goals as a multi-column vindex. Moreover, you avoid having to define additional vindexes for query routing.
+
+The downside of this approach is that it is harder to migrate an id to a different region.
+
+## Reference Tables
+
+Sharded databases often need the ability to join their tables with smaller “reference” tables. For example, the `product` table could be seen as a reference table. Other use cases are tables that map static information like zipcode to city, etc.
+
+Joining against these tables across keyspaces results in cross-shard joins that may not be very efficient or fast.
+
+Vitess allows you to create a table in a sharded keyspace as a reference table. This means that it will treat the table as having an identical set of rows across all shards. A query that joins a sharded table against such reference tables is then performed locally within each shard.
+
+A reference table should not have any vindex, and is defined in the VSchema as a reference type:
+
+```json
+{
+ "sharded": true,
+ "tables": {
+ "zip_detail": { "type": "reference" }
+ }
+}
+```
+
+It may become a challenge to keep a reference table correctly updated across all shards. Vitess supports the [Materialize](../../migration/materialize) feature that allows you to maintain the original table in an unsharded keyspace and automatically propagate changes to that table in real-time across all shards.
+
+## Column List
+
+The VSchema allows you to specify the list of columns along with their types for every table. This allows Vitess to make optimization decisions where necessary.
+
+For example, specifying that a column contains text allows VTGate to request further collation specific information (`weight_string`) if additional sorting is needed after collecting results from all shards.
+
+For example, issuing this query against `customer` would fail:
+
+```text
+mysql> select customer_id, uname from customer order by uname;
+ERROR 1105 (HY000): vtgate: http://sougou-lap1:12345/: types are not comparable: VARCHAR vs VARCHAR
+```
+
+However, we can modify the VSchema as follows:
+
+```json
+ "customer": {
+ "column_vindexes": [{
+ "column": "customer_id",
+ "name": "hash"
+ }],
+ "auto_increment": {
+ "column": "customer_id",
+ "sequence": "product.customer_seq"
+ },
+ "columns": [{
+ "name": "uname",
+ "type": "VARCHAR"
+ }]
+ }
+```
+
+Re-issuing the same query will now succeed:
+
+```text
+mysql> select customer_id, uname from customer order by uname;
++-------------+---------+
+| customer_id | uname |
++-------------+---------+
+| 1 | alice |
+| 2 | bob |
+| 3 | charlie |
+| 4 | dan |
+| 5 | eve |
++-------------+---------+
+5 rows in set (0.00 sec)
+```
+
+Specifying columns against tables also allows VTGate to resolve ambiguous naming of columns against the right tables.
+
+#### Authoritative List
+
+If you have listed all columns of a table in the VSchema, you can add the `column_list_authoritative` flag to the table:
+
+```json
+ "customer": {
+ "column_vindexes": [{
+ "column": "customer_id",
+ "name": "hash"
+ }],
+ "auto_increment": {
+ "column": "customer_id",
+ "sequence": "product.customer_seq"
+ },
+ "columns": [{
+ "name": "uname",
+ "type": "VARCHAR"
+ }],
+ "column_list_authoritative": true
+ }
+```
+
+This flag causes VTGate to automatically expand expressions like `select *` or insert statements that don’t specify the column list.
+
+The caveat about using this feature is that you have to keep this column list in sync with the underlying schema.
+
+In the future, Vitess will allow you to pull this information from the vttablets and automatically keep it up-to-date.
+
+## Routing Rules
+
+Routing Rules are an advanced method of redirecting queries meant for one table to another. They are just pointers and are analogous to symbolic links in a file system. You should generally not have to use routing rules in Vitess.
+
+Workflows like `MoveTables` make use of routing rules to create the existence of the target tables and manage traffic switch from source to target by manipulating these routing rules.
+
+For more information, please refer to the [Routing Rules](../../../reference/features/schema-routing-rules) section.
+
diff --git a/content/en/docs/user-guides/vschema-guide/img/vschema1.png b/content/en/docs/user-guides/vschema-guide/img/vschema1.png
new file mode 100644
index 000000000..f85138de2
Binary files /dev/null and b/content/en/docs/user-guides/vschema-guide/img/vschema1.png differ
diff --git a/content/en/docs/user-guides/vschema-guide/img/vschema2.png b/content/en/docs/user-guides/vschema-guide/img/vschema2.png
new file mode 100644
index 000000000..85ec159f2
Binary files /dev/null and b/content/en/docs/user-guides/vschema-guide/img/vschema2.png differ
diff --git a/content/en/docs/user-guides/vschema-guide/lookup-as-primary.md b/content/en/docs/user-guides/vschema-guide/lookup-as-primary.md
new file mode 100644
index 000000000..cc84f6378
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/lookup-as-primary.md
@@ -0,0 +1,146 @@
+---
+title: Lookup as Primary Vindex
+weight: 10
+---
+
+It is likely that a customer order goes through a life cycle of events. This would best be represented in a separate `corder_event` table that will contain a `corder_id` column as a foreign key into `corder.corder_id`. It would also be beneficial to co-locate the event rows with their associated order.
+
+Just like we shared the `hash` vindex between `customer` and `corder`, we can share `corder_keyspace_idx` between `corder` and `corder_event`. We can also make it the Primary Vindex for `corder_event`. When an order is created, the lookup row for it is also created. Subsequently, an insert into `corder_event` will request the vindex to compute the `keyspace_id` for that `corder_id`, and that will succeed because the lookup entry for it already exists. This is where the significance of the owner table comes into play: The owner table creates the entries, whereas other tables only read those entries.
+
+Inserting a `corder_event` row without creating a corresponding `corder` entry will result in an error. This behavior is in line with the traditional foreign key constraint enforced by relational databases.
+
+Sharing the lookup vindex also has the additional benefit of saving space because we avoid creating separate entries for the new table.
+
+We start with creating the sequence table in the `product` keyspace.
+
+Schema:
+
+```sql
+create table corder_event_seq(id bigint, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';
+insert into corder_event_seq(id, next_id, cache) values(0, 1, 3);
+```
+
+VSchema:
+
+```json
+ "corder_event_seq": { "type": "sequence" }
+```
+
+We then create the `corder_event` table in `customer`:
+
+```sql
+create table corder_event(corder_event_id bigint, corder_id bigint, ename varchar(128), primary key(corder_id, corder_event_id));
+```
+
+In the VSchema, there is no need to create a vindex because we are going to reuse the existing one:
+
+```json
+ "corder_event": {
+ "column_vindexes": [{
+ "column": "corder_id",
+ "name": "corder_keyspace_idx"
+ }],
+ "auto_increment": {
+ "column": "corder_event_id",
+ "sequence": "product.corder_event_seq"
+ }
+ }
+```
+
+Alternate VSchema DDL:
+
+```sql
+alter vschema add sequence product.corder_event_seq;
+alter vschema on customer.corder_event add vindex corder_keyspace_idx(corder_id);
+alter vschema on customer.corder_event add auto_increment corder_event_id using product.corder_event_seq;
+```
+
+We can now insert rows in `corder_event` against rows in `corder`:
+
+```text
+mysql> insert into corder(customer_id, product_id, oname) values (1,1,'gift'),(1,2,'gift'),(2,1,'work'),(3,2,'personal'),(4,1,'personal');
+Query OK, 5 rows affected (0.04 sec)
+
+mysql> insert into corder_event(corder_id, ename) values(1, 'paid'), (5, 'delivered');
+Query OK, 2 rows affected (0.01 sec)
+
+mysql> insert into corder_event(corder_id, ename) values(6, 'expect failure');
+ERROR 1105 (HY000): vtgate: http://sougou-lap1:12345/: execInsertSharded: getInsertShardedRoute: could not map [INT64(6)] to a keyspace id
+```
+
+As expected, inserting a row for a non-existent order results in an error.
+
+### Reversible Vindexes
+
+In Vitess, it is insufficient for a table to only have a Lookup Vindex. This is because it is not practical to reshard such a table. The overhead of performing a lookup before redirecting every row event to a new shard would be prohibitively expensive.
+
+To overcome this limitation, we must add a column with a non-lookup vindex, also known as Functional Vindex to the table. By rule, the Primary Vindex computes the keyspace id of the row. This means that the value of the column should also be such that it yields the same keyspace id.
+
+A Reversible Vindex is one that can back-compute the column value from a given keyspace id. If such a vindex is used for this new column, then Vitess will automatically perform this work and fill the correct value for it. The list of vindex properties, like Functional, Reversible, etc. are listed in the [Vindexes Reference](../../../features/vindexes).
+
+In other words, adding a column with a vindex that is both Functional and Reversible allows Vitess to auto-fill the values, thereby avoiding any impact to the application logic.
+
+The `binary` vindex is one that yields the input value itself as the `keyspace_id`, and is naturally reversible. Using this Vindex will generate the `keyspace_id` as the column value. The modified schema for the table will be as follows:
+
+```sql
+create table corder_event(corder_event_id bigint, corder_id bigint, ename varchar(128), keyspace_id varbinary(10), primary key(corder_id, corder_event_id));
+```
+
+We create a vindex instantiation for `binary`:
+
+```json
+ "binary": {
+ "type": "binary"
+ }
+```
+
+Modify the table VSchema:
+
+```json
+ "corder_event": {
+ "column_vindexes": [{
+ "column": "corder_id",
+ "name": "corder_keyspace_idx"
+ }, {
+ "column": "keyspace_id",
+ "name": "binary"
+ }],
+ "auto_increment": {
+ "column": "corder_event_id",
+ "sequence": "product.corder_event_seq"
+ }
+ }
+```
+
+Alternate VSchema DDL:
+
+```sql
+alter vschema on customer.corder_event add vindex `binary`(keyspace_id) using `binary`;
+```
+
+Note that `binary` needs to be backticked because it is a keyword.
+
+After these modifications, we can now observe that the `keyspace_id` column is getting automatically populated:
+
+```text
+mysql> insert into corder(customer_id, product_id, oname) values (1,1,'gift'),(1,2,'gift'),(2,1,'work'),(3,2,'personal'),(4,1,'personal');
+Query OK, 5 rows affected (0.01 sec)
+
+mysql> insert into corder_event(corder_id, ename) values(1, 'paid'), (5, 'delivered');
+Query OK, 2 rows affected (0.01 sec)
+
+mysql> select corder_event_id, corder_id, ename, hex(keyspace_id) from corder_event;
++-----------------+-----------+-----------+------------------+
+| corder_event_id | corder_id | ename | hex(keyspace_id) |
++-----------------+-----------+-----------+------------------+
+| 1 | 1 | paid | 166B40B44ABA4BD6 |
+| 2 | 5 | delivered | D2FD8867D50D2DFE |
++-----------------+-----------+-----------+------------------+
+2 rows in set (0.00 sec)
+```
+
+There is no support for backfilling the reversible vindex column yet. This will be added soon.
+
+{{< info >}}
+The original `keyspace_id` for all these rows came from `customer_id`. Since `hash` is also a reversible vindex, reversing the `keyspace_id` using `hash` will yield the `customer_id`. We could instead leverage this knowledge to replace `keyspace_id+binary` with `customer_id+hash`. Vitess will auto-populate the correct value. Using this approach may be more beneficial because `customer_id` is a value the application can understand and make use of.
+{{< /info >}}
diff --git a/content/en/docs/user-guides/vschema-guide/non-unique-lookup.md b/content/en/docs/user-guides/vschema-guide/non-unique-lookup.md
new file mode 100644
index 000000000..59526f9f7
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/non-unique-lookup.md
@@ -0,0 +1,194 @@
+---
+title: Non-Unique Lookup Vindexes
+weight: 9
+---
+
+The `oname` column in `corder` can contain duplicate values. There may be a need in the application to frequently search by this column:
+
+```sql
+select * from corder where oname='gift'
+```
+
+To prevent this query from resulting in a full scatter, we will need to create a lookup vindex for it. But this time, it will need to be non-unique. However, the fact that duplicates are allowed leads to a complication with the lookup table approach. Let us look at the insert query:
+
+```sql
+insert into corder(customer_id, product_id, oname) values (1,1,'gift'),(1,2,'gift'),(2,1,'work'),(3,2,'personal'),(4,1,'personal');
+```
+
+We see that `customer_id 1` has two rows where the `oname` is `gift`. If we try to create entries for those two in a lookup table, they would be identical:
+
+```text
++-----------+--------------+
+| oname | hex(keyspace_id) |
++-----------+--------------+
+| gift | 166B40B44ABA4BD6 | (corder_id=1)
+| gift | 166B40B44ABA4BD6 | (corder_id=2)
++-----------+--------------+
+```
+
+To disambiguate this situation, non-unique lookup vindexes require you to add additional columns to the lookup table. They are typically the Primary Key of the main table. For the sake of demonstration, let us create this as a sharded table in the `customer` keyspace:
+
+```sql
+create table oname_keyspace_idx(oname varchar(128), corder_id bigint, keyspace_id varbinary(10), primary key(oname, corder_id));
+```
+
+Note that the primary key includes the `oname` column as well as the `corder_id` column.
+
+Because `oname` is a text column, the recommended Primary Vindex for it would be `unicode_loose_md5`, which also requires a vindex instantiation:
+
+“vindexes” section:
+
+```json
+ "unicode_loose_md5": {
+ "type": "unicode_loose_md5"
+ }
+```
+
+“tables” section:
+
+```json
+ "oname_keyspace_idx": {
+ "column_vindexes": [{
+ "column": "oname",
+ "name": "unicode_loose_md5"
+ }]
+ }
+```
+
+The lookup vindex should reference these new columns as follows:
+
+```json
+ "oname_keyspace_idx": {
+ "type": "consistent_lookup",
+ "params": {
+ "table": "customer.oname_keyspace_idx",
+ "from": "oname,corder_id",
+ "to": "keyspace_id"
+ },
+ "owner": "corder"
+ }
+```
+
+{{< info >}}
+This Vindex could also be seen as a multi-column Unique Lookup Vindex: For a given pair of `oname,corder_id` as input, the result can only yield a single `keyspace_id`. However, the `consistent_lookup` vindex functionality only supports resolution using the first column `oname`. In the future, we may add the ability to use both columns as input if they are present in the `where` clause. This may result in the merger of `consistent_lookup` with a multi-column version of `consistent_lookup_unique` that can also perform non-unique lookups on a subset of the inputs.
+{{< /info >}}
+
+Finally, we tie the associated columns in `corder` to the vindex:
+
+```json
+ "corder": {
+ "column_vindexes": [{
+ "column": "customer_id",
+ "name": "hash"
+ }, {
+ "column": "corder_id",
+ "name": "corder_keyspace_idx"
+ }, {
+ "columns": ["oname", "corder_id"],
+ "name": "oname_keyspace_idx"
+ }],
+ "auto_increment": {
+ "column": "corder_id",
+ "sequence": "product.corder_seq"
+ }
+ }
+```
+
+Alternate VSchema DDL:
+
+```sql
+alter vschema on customer.oname_keyspace_idx add vindex unicode_loose_md5(oname) using unicode_loose_md5;
+alter vschema on customer.corder add vindex oname_keyspace_idx(oname,corder_id) using consistent_lookup with owner=`corder`, table=`customer.oname_keyspace_idx`, from=`oname,corder_id`, to=`keyspace_id`;
+```
+
+We can now look at the effects of this change:
+
+```text
+mysql> insert into corder(customer_id, product_id, oname) values (1,1,'gift'),(1,2,'gift'),(2,1,'work'),(3,2,'personal'),(4,1,'personal');
+Query OK, 5 rows affected (0.03 sec)
+
+mysql> use `customer:-80`;
+Database changed
+mysql> select oname, corder_id, hex(keyspace_id) from oname_keyspace_idx;
++-------+-----------+------------------+
+| oname | corder_id | hex(keyspace_id) |
++-------+-----------+------------------+
+| gift | 1 | 166B40B44ABA4BD6 |
+| gift | 2 | 166B40B44ABA4BD6 |
+| work | 3 | 06E7EA22CE92708F |
++-------+-----------+------------------+
+3 rows in set (0.00 sec)
+
+mysql> use `customer:80-`;
+Database changed
+mysql> select oname, corder_id, hex(keyspace_id) from oname_keyspace_idx;
++----------+-----------+------------------+
+| oname | corder_id | hex(keyspace_id) |
++----------+-----------+------------------+
+| personal | 4 | 4EB190C9A2FA169C |
+| personal | 5 | D2FD8867D50D2DFE |
++----------+-----------+------------------+
+2 rows in set (0.00 sec)
+```
+
+We can see that the lookup table is following its own sharding scheme and distributing its rows according to the value of the `oname` column.
+
+Deleting one of the `corder` rows results in the corresponding lookup row being deleted:
+
+```text
+mysql> delete from corder where corder_id=1;
+Query OK, 1 row affected (0.00 sec)
+
+mysql> select oname, corder_id, hex(keyspace_id) from oname_keyspace_idx where oname='gift';
++-------+-----------+------------------+
+| oname | corder_id | hex(keyspace_id) |
++-------+-----------+------------------+
+| gift | 2 | 166B40B44ABA4BD6 |
++-------+-----------+------------------+
+1 row in set (0.00 sec)
+```
+
+{{< info >}}
+You would typically have to create a MySQL non-unique index on `oname` for queries to work efficiently. While these vindexes and indexes improve read performance, the trade-off is that they also increase storage requirements and amplify writes when inserting rows.
+{{< /info >}}
+
+### CreateLookupVindex
+
+To create such a lookup vindex on a real Vitess cluster, you can use the following instructions:
+
+Save the following json into a file, say `oname_keyspace_idx.json`:
+
+```json
+{
+ "sharded": true,
+ "vindexes": {
+ "oname_keyspace_idx": {
+ "type": "consistent_lookup",
+ "params": {
+ "table": "customer.oname_keyspace_idx",
+ "from": "oname,corder_id",
+ "to": "keyspace_id"
+ },
+ "owner": "corder"
+ }
+ },
+ "tables": {
+ "corder": {
+ "column_vindexes": [{
+ "columns": ["oname", "corder_id"],
+ "name": "oname_keyspace_idx"
+ }]
+ }
+ }
+}
+```
+
+And issue the vtctlclient command:
+
+```sh
+$ vtctlclient -server CreateLookupVindex -tablet_types=REPLICA customer "$(cat oname_keyspace_idx.json)"
+```
+
+The workflow will automatically create the necessary Primary Vindex entries for `oname_keyspace_idx` knowing that it is sharded.
+
+After the backfill is done, you should clean up the workflow. More detailed instructions are available in the [CreateLookupVindex Reference](../../configuration-advanced/createlookupvindex)
diff --git a/content/en/docs/user-guides/vschema-guide/overview.md b/content/en/docs/user-guides/vschema-guide/overview.md
new file mode 100644
index 000000000..00cfd877d
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/overview.md
@@ -0,0 +1,34 @@
+---
+title: Overview
+weight: 2
+---
+
+One of the goals for Vitess is to provide a unified view for a large number of MySQL clusters distributed across multiple data centers and regions.
+
+Vitess achieves this goal by allowing the application to connect to any VTGate server, and that server gives you the semblance of being connected to a single MySQL server. The metadata that maps the logical view to the physical MySQL servers is stored in the topology.
+
+In this logical view, a Vitess keyspace is the equivalent of a MySQL database. In many cases, this is a one-to-one mapping where a keyspace directly corresponds to a physical MySQL server with a single database. However, a Vitess keyspace can also be sharded. If so, a single keyspace would map to multiple MySQL servers behind the scenes.
+
+The topology is typically spread across multiple Topo Servers: The Global Topo server contains global information, like the list of keyspaces, shards and cells. This information gets deployed into cell-specific topo servers. Each cell-specific Topo Server contains additional information about vttablets and MySQL servers running in that cell. With this architecture, an outage in one cell does not affect other cells.
+
+The topo also stores a VSchema for each keyspace. For an unsharded keyspace, the vschema is a simple list of table names. If a keyspace is sharded, then it must contain additional metadata about the sharding scheme for each table, and how they relate to each other. When a query is received by VTGate, the information in the vschema is used to make decisions about how to serve the query. In some cases, it will result in the query being routed to a single shard. In other cases, it could result in the query being sent to all shards, etc.
+
+This guide explains how to build vschemas for Vitess keyspaces.
+
+### Demo
+
+To illustrate the various features of the VSchema, we will make use of the [demo app](https://github.com/vitessio/vitess/tree/master/examples/demo). After installing Vitess, you can launch this demo by running `go run demo.go`. Following this, you can visit http://localhost:8000 to view the tables, issue arbitrary queries, and view their effects.
+
+Alternatively, you can also connect to Vitess using a MySQL client: `mysql -h 127.0.0.1 -P 12348`.
+
+The demo models a set of tables that are similar to those presented in the [Getting Started](../../../get-started/local) guide, but with more focus on the VSchema.
+
+Note that the demo brings up a test process called vtcombo (instead of a real Vitess cluster), which is functionally equivalent to all the components of Vitess, but within a single process.
+
+You can also use the demo app to follow along the steps of this user guide. If so, you can start by emptying out the files under `schema/product` and `schema/customer`, and incrementally making the changes presented in the steps that follow.
+
+### VSchema DDL
+
+The demo describes the VSchema JSON syntax. Many of the changes can be executed by issuing special DDL commands that Vitess understands. Wherever applicable, we have provided the equivalent DDL construct you could apply if you were running a live system. All the DDLs are also listed in the `vschema_ddls.sql` file.
+
+It is generally recommended that you get familiar with the JSON syntax as it will be useful for troubleshooting if something does not work as intended.
diff --git a/content/en/docs/user-guides/vschema-guide/pictorial.md b/content/en/docs/user-guides/vschema-guide/pictorial.md
new file mode 100644
index 000000000..f3380f47b
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/pictorial.md
@@ -0,0 +1,14 @@
+---
+title: Pictorial Summary
+weight: 12
+---
+
+The following two diagrams highlight some of the relationships that exist between VSchema elements and the MySQL tables.
+
+### product and customer
+
+
+
+### corder
+
+
diff --git a/content/en/docs/user-guides/vschema-guide/sequences.md b/content/en/docs/user-guides/vschema-guide/sequences.md
new file mode 100644
index 000000000..7b5322659
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/sequences.md
@@ -0,0 +1,95 @@
+---
+title: Sequences
+weight: 6
+---
+
+The sharded `customer` table we created did not have an auto-increment column. The Vitess Sequence feature can be used to emulate the same behavior as MySQL’s auto-increment. A Vitess sequence is a single row unsharded tablet that keeps track of ids issued so far. Additionally, a configurable number of values can be cached by vttablet to minimize round trips into MySQL.
+
+We will create the sequence table in the unsharded `product` keyspace as follows:
+
+```sql
+create table customer_seq(id bigint, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';
+insert into customer_seq(id, next_id, cache) values(0, 1, 3);
+```
+
+Note the special comment `vitess_sequence`. This instructs vttablet that this is a special table.
+
+The table needs to be pre-populated with a single row where:
+* `id` must always be 0
+* `next_id` should be set to the next (starting) value of the sequence
+* `cache` is the number of values to cache before updating the table for the next value. This value should be set to a fairly large number like 1000. We have set the value to `3` mainly to demonstrate how the feature works.
+
+Since this is a special table, we have to inform the vschema by giving it a `sequence` type.
+
+```
+ "customer_seq": { "type": "sequence" }
+```
+
+Once setup this way, you can use the special `select next` syntax to generate values from this sequence:
+
+
+```text
+mysql> select next 2 values from customer_seq;
++---------+
+| nextval |
++---------+
+| 1 |
++---------+
+1 row in set (0.00 sec)
+
+mysql> select next 1 values from customer_seq;
++---------+
+| nextval |
++---------+
+| 3 |
++---------+
+1 row in set (0.00 sec)
+```
+
+The construct returns the first of the N values generated.
+
+However, this is insufficient to emulate MySQL’s auto-increment behavior. To achieve this, we have to inform the VSchema that the `customer_id` column should use this sequence to generate values if no value is specified. This is done by adding the following section to the `customer` table:
+
+```json
+ "auto_increment": {
+ "column": "customer_id",
+ "sequence": "product.customer_seq"
+ }
+```
+
+Alternate VSchema DDL:
+
+```sql
+alter vschema add sequence product.customer_seq;
+alter vschema on customer.customer add auto_increment customer_id using product.customer_seq;
+```
+
+With this, you can insert into `customer` without specifying the `customer_id`:
+
+```text
+mysql> insert into customer(uname) values('alice'),('bob'),('charlie'),('dan'),('eve');
+Query OK, 5 rows affected (0.03 sec)
+
+mysql> use `customer:-80`;
+Database changed
+mysql> select * from customer;
++-------------+---------+
+| customer_id | uname |
++-------------+---------+
+| 1 | alice |
+| 2 | bob |
+| 3 | charlie |
+| 5 | eve |
++-------------+---------+
+4 rows in set (0.00 sec)
+
+mysql> use `customer:80-`;
+Database changed
+mysql> select * from customer;
++-------------+-------+
+| customer_id | uname |
++-------------+-------+
+| 4 | dan |
++-------------+-------+
+1 row in set (0.00 sec)
+```
diff --git a/content/en/docs/user-guides/vschema-guide/sharded.md b/content/en/docs/user-guides/vschema-guide/sharded.md
new file mode 100644
index 000000000..e0a2c83bf
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/sharded.md
@@ -0,0 +1,130 @@
+---
+title: Sharded Keyspace
+weight: 5
+---
+
+A sharded keyspace allows you to split a large database into smaller parts by distributing the rows of each table into different shards. In Vitess, each shard is assigned a `keyrange`. Every row has a keyspace id, and this value decides the shard in which the row lives. For key-value stores, the keyspace id is dictated by the value of the key, also known as the sharding key. In Vitess, this is known as the Primary Vindex. But it differs from a sharding key in the following ways:
+
+* Any column or a set of columns can be chosen to be the primary vindex.
+* The Vindex also decides the sharding function that controls how the data is distributed.
+* The sharding function is pluggable, allowing for user-defined sharding schemes.
+
+Vitess provides many predefined vindex types. The most popular ones are:
+* `hash`: for numbers
+* `unicode_loose_md5`: for text columns
+* `binary_md5`: for binary columns
+
+In our example, we are going to designate `customer` as a sharded keyspace, and create a `customer` table in it. The schema for the table is as follows:
+
+```sql
+create table customer(customer_id bigint, uname varchar(128), primary key(customer_id));
+```
+
+In the VSchema, we need to designate which column should be the Primary Vindex, and choose the vindex type for it. The `customer_id` column seems to be the natural choice. Since it is a number, we will choose `hash` as the vindex type:
+
+```json
+{
+ "sharded": true,
+ "vindexes": {
+ "hash": {
+ "type": "hash"
+ }
+ },
+ "tables": {
+ "customer": {
+ "column_vindexes": [{
+ "column": "customer_id",
+ "name": "hash"
+ }]
+ }
+ }
+}
+```
+
+In the above section, we are instantiating a vindex named `hash` from the vindex type `hash`. Such instantiations are listed in the `vindexes` section of the vschema. The tables are expected to refer to the instantiated name. There are a few reasons why this additional level of indirection is necessary:
+* As we will see later, vindexes can be instantiated with different input parameters. In such cases, they have to have their own distinct names.
+* Vindexes can be shared by tables, and this has special meaning. We will cover this in a later section.
+* Vindexes can also be referenced as if they were tables and can be used to compute the keyspace id for a given input.
+
+The `column_vindexes` section is a list. This is because a table can have multiple vindexes. If so, the first vindex in the list must be the Primary Vindex. More information about vindexes can be found in the [Vindex Reference](../../../reference/features/vindexes).
+
+Alternate VSchema DDL:
+
+```sql
+alter vschema on customer.customer add vindex hash(customer_id) using hash;
+```
+
+The DDL creates the `hash` vindex under the `vindexes` section, the `customer` table under the `tables` section, and associates the `customer_id` column to `hash`. For sharded keyspaces, the only way to create a table is using the above construct. This is because a primary vindex is mandatory for sharded tables.
+
+{{< info >}}
+Every sharded table must have a Primary Vindex. A Primary Vindex must be instantiated from a vindex type that is Unique. `hash`, `unicode_loose_md5` and `binary_md5` are unique vindex types.
+{{< /info >}}
+
+The demo brings up the customer as two shards: `-80` and `80-`. For a `hash` vindex, input values of 1, 2 and 3 fall in the `-80` range, and 4 falls in the `80-` range. Restarting the demo with the updated configs should allow you to perform the following:
+
+```text
+mysql> insert into customer(customer_id,uname) values(1,'alice'),(4,'dan');
+Query OK, 2 rows affected (0.00 sec)
+
+mysql> use `customer:-80`;
+Database changed
+mysql> select * from customer;
++-------------+-------+
+| customer_id | uname |
++-------------+-------+
+| 1 | alice |
++-------------+-------+
+1 row in set (0.00 sec)
+
+mysql> use `customer:80-`;
+Database changed
+mysql> select * from customer;
++-------------+-------+
+| customer_id | uname |
++-------------+-------+
+| 4 | dan |
++-------------+-------+
+1 row in set (0.00 sec)
+```
+
+You will notice that we used a special shard targeting construct: `use customer:-80`. Vitess allows you to use this hidden database name to bypass its routing logic and directly send queries to a specific shard. Using this construct, we are able to verify that the rows went to different shards.
+
+At the time of insert, the Primary Vindex is used to compute and assign a keyspace id to each row. This keyspace id gets used to decide where the row will be stored. Although a keyspace id is not explicitly stored anywhere, it must be seen as an unchanging property of that row, as if there was an invisible column for it.
+
+Consequently, you cannot make changes to a row that can cause the keyspace id to change. Such a change will be supported in the future through a shard move operation. Trying to change the value of a Primary Vindex results in an error:
+
+```text
+mysql> update customer set customer_id=2 where customer_id=1;
+ERROR 1235 (HY000): vtgate: http://sougou-lap1:12345/: unsupported: You can't update primary vindex columns. Invalid update on vindex: hash
+```
+
+A Primary Vindex can also be used to find rows if referenced in a where clause:
+
+```text
+mysql> select * from customer where customer_id=1;
++-------------+-------+
+| customer_id | uname |
++-------------+-------+
+| 1 | alice |
++-------------+-------+
+1 row in set (0.00 sec)
+```
+
+If you run the above query in the demo app, the panel on the bottom right will show that the query was executed only on one shard.
+
+On the other hand, the query below will get sent to all shards because there is no where clause:
+
+```text
+mysql> select * from customer;
++-------------+-------+
+| customer_id | uname |
++-------------+-------+
+| 4 | dan |
+| 1 | alice |
++-------------+-------+
+2 rows in set (0.01 sec)
+```
+
+{{< info >}}
+There is no implicit or predictable ordering for rows that are gathered from multiple shards. If a specific order is required, the query must include an `order by` clause.
+{{< /info >}}
diff --git a/content/en/docs/user-guides/vschema-guide/sharding-guidelines.md b/content/en/docs/user-guides/vschema-guide/sharding-guidelines.md
new file mode 100644
index 000000000..13dd68c87
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/sharding-guidelines.md
@@ -0,0 +1,79 @@
+---
+title: Sharding Guidelines
+weight: 3
+---
+
+The following guidelines are not set in stone. They mainly establish a framework for making decisions.
+
+### Why
+
+There was a time when sharding used to be a line that one should avoid crossing for as long as possible. However, with Vitess considerably reducing the pain of sharding, we can look at leveraging some of its benefits much sooner than when a machine runs out of capacity:
+
+* Smaller blast radius: If a shard goes down, the outage affects a smaller percentage of the users.
+* Improved resource utilization: It is difficult to pack large instances of servers efficiently across machines. It is much easier to utilize the capacity of the existing hardware if the shard sizes were relatively small. Orchestration systems like Kubernetes further facilitate such utilization.
+* Reduced contention: MySQL itself runs a lot better when instance sizes are small. There is less internal contention, replicas tend to keep up more easily with their master, etc.
+* Improved maintenance: Operations like schema deployment can happen in parallel across all shards and finish much sooner than usual.
+
+There is a general worry that the complexity of deployment increases with the number of servers. However, this becomes a static cost once the necessary automation and monitoring is put in place.
+
+### Why not
+
+There are also reasons why you may want to avoid sharding. The main reason is that you may introduce inefficiencies by splitting data that would have been better off if it stayed together. Or, if your database is extremely small.
+
+However, if you reach a point where the data is starting to grow, sharding may become inevitable.
+
+### Moving Tables
+
+Typically, the first step you may perform is to split your database by moving some tables on to other databases. In Vitess parlance, we call this as splitting off keyspaces. The [MoveTables](../../migration/move-tables) workflow allows you to perform this with minimal impact to the application.
+
+### Resharding
+
+Beyond a certain point, it may not make sense to separate tables that are strongly related to each other. This is when resharding comes into play. Choosing the “sharding key” is often intuitively obvious.
+
+If you analyze the query pattern in the application, the query with the highest QPS will dictate the sharding key (or Primary Vindex). In our example below, we will be choosing `customer_id` as the Primary Vindex for the `customer` table.
+
+If there are queries with other where clauses on the same table, those would be candidates for secondary lookup vindexes.
+
+### Joins
+
+The next criteria to take into account are joins. If you are performing joins across tables, it will be beneficial to keep those rows together. In our example, we will be keeping the rows of the order table along with their customer. This grouping will allow us to efficiently perform operations like reading all orders placed by a customer.
+
+### Transactions
+
+It is important to keep transactions to be within one shard. Vitess currently does not guarantee atomicity for transactions that go across shards. Grouping related rows together usually results in transactions also falling within the same shard.
+
+But there are situations where this may not be possible. If so, you can see if it will be possible to avoid this problem by grouping data differently. However, this may not be possible either. A well known use case is one where customers send each other money.
+
+In such situations, you can look at refactoring the application such that transactions are broken into smaller single shard transactions.
+
+Vitess will be adding support for distributed transactions soon.
+
+### Large Tenants
+
+If your application is tenant-based, it is possible that a single tenant may grow so big that they may not fit in one shard. If so, it is likely that the application is using a different key that has a higher cardinality than the tenant id.
+
+The question to ask oneself is: if the tenant were a single application by themselves, what would be their sharding key, and then shard by that key instead of the tenant id.
+
+Vitess has started rolling out support for multi-column Vindexes. Once this feature is fully done, you will be able to shard by the tenant id and a secondary key. The two-column sharding approach will allow you to group all data for a given tenant into a smaller set of shards rather than a random distribution. This may be beneficial for security or compliance reasons, in case the tenant would want their data to be physically isolated from other tenants.
+
+### Region Sharding
+
+The Vitess multi-column Vindex feature also allows you to locate data associated with regions in different geographical locations. For more details, see [Region-based Sharding](../../configuration-advanced/region-sharding).
+
+### Many-to-Many relationships
+
+Sharding works well only if the relationship between data is hierarchical (one-to-one or one-to-many). If a table has foreign keys into multiple other tables, you have to choose one based on the strongest relationship and group the rows by that foreign key. The rest of the relationships will incur cross-shard overheads.
+
+If more than one relationship is critically strong, you can look at the [Materialization](../../../reference/vreplication/materialize) feature which allows you to create a materialized view of the table that is sharded using the other relationship. The application will write to the source, and the other view will be automatically updated.
+
+### Course Correction
+
+It may happen that the original sharding decision is not working out. It may also be possible that the application evolves in such a way that the sharding decision you previously made does not make sense any more.
+
+In such cases, the [MoveTables](../../migration/move-tables) feature can be used to change the sharding key of existing tables.
+
+Essentially, Vitess removes the fear of making the wrong sharding decision because you can always change your mind later.
+
+### Thumb Rule
+
+Although a Vitess shard can grow to many terabytes, we have seen that 250GB is the sweet spot. If your data size approaches this limit, it is time to think about splitting your data.
diff --git a/content/en/docs/user-guides/vschema-guide/shared-vindexes.md b/content/en/docs/user-guides/vschema-guide/shared-vindexes.md
new file mode 100644
index 000000000..2175b0781
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/shared-vindexes.md
@@ -0,0 +1,172 @@
+---
+title: Shared Vindexes and Foreign Keys
+weight: 7
+---
+
+Let us now look at creating the `corder` table that will contain orders placed by the customers. It will be beneficial to group the rows of the orders in the same shard as that of the customer that placed the orders. Doing things this way will allow for simpler join queries between `customer` and `corder`. There will also be transactional benefits: any transaction that also updates the customer row along with an order will be a single shard transaction.
+
+To make this happen in Vitess, all you have to do is specify that `corder.customer_id` uses the `hash` vindex, which is the same one used by `customer.customer_id`.
+
+This is one situation where a Primary Vindex conceptually differs from a traditional database Primary Key. Whereas a Primary Key makes a row unique, a Vitess Primary Vindex only yields a Unique value. But multiple rows with the same Primary Vindex value can exist.
+
+In other words, the Primary Vindex column need not be the primary key, or unique within MySQL. This is convenient for the `corder` table because we want customers to place multiple orders. In this case, all orders placed by a customer will have the same `customer_id`. The Primary Vindex for those will yield the same keyspace id as that of the customer. Therefore, all the rows for that customer’s orders will end up in the same shard along with the customer row.
+
+Since `corder` rows will need to have their own unique identifier, we also need to create a separate sequence for it in the product keyspace.
+
+```sql
+create table corder_seq(id bigint, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';
+insert into corder_seq(id, next_id, cache) values(0, 1, 3);
+```
+
+VSchema:
+
+```json
+ "corder_seq": { "type": "sequence" }
+```
+
+We create the `corder` table as follows:
+
+```sql
+create table corder(corder_id bigint, customer_id bigint, product_id bigint, oname varchar(128), primary key(corder_id));
+```
+
+VSchema:
+```json
+ "corder": {
+ "column_vindexes": [{
+ "column": "customer_id",
+ "name": "hash"
+ }],
+ "auto_increment": {
+ "column": "corder_id",
+ "sequence": "product.corder_seq"
+ }
+ }
+```
+
+Alternate VSchema DDL:
+
+```sql
+alter vschema on customer.corder add vindex hash(customer_id);
+alter vschema add sequence product.corder_seq;
+alter vschema on customer.corder add auto_increment corder_id using product.corder_seq;
+```
+
+Inserting into `corder` yields the following results:
+
+```text
+mysql> insert into corder(customer_id, product_id, oname) values (1,1,'gift'),(1,2,'gift'),(2,1,'work'),(3,2,'personal'),(4,1,'personal');
+Query OK, 5 rows affected (0.03 sec)
+
+mysql> use `customer:-80`;
+Database changed
+mysql> select * from corder;
++-----------+-------------+------------+----------+
+| corder_id | customer_id | product_id | oname |
++-----------+-------------+------------+----------+
+| 1 | 1 | 1 | gift |
+| 2 | 1 | 2 | gift |
+| 3 | 2 | 1 | work |
+| 4 | 3 | 2 | personal |
++-----------+-------------+------------+----------+
+4 rows in set (0.00 sec)
+
+mysql> use `customer:80-`;
+Database changed
+mysql> select * from corder;
++-----------+-------------+------------+----------+
+| corder_id | customer_id | product_id | oname |
++-----------+-------------+------------+----------+
+| 5 | 4 | 1 | personal |
++-----------+-------------+------------+----------+
+1 row in set (0.00 sec)
+```
+
+As expected, orders are created in the same shard as their customer. Selecting orders by their customer id goes to a single shard:
+
+```text
+mysql> select * from corder where customer_id=1;
++-----------+-------------+------------+-------+
+| corder_id | customer_id | product_id | oname |
++-----------+-------------+------------+-------+
+| 1 | 1 | 1 | gift |
+| 2 | 1 | 2 | gift |
++-----------+-------------+------------+-------+
+2 rows in set (0.00 sec)
+```
+
+Joining `corder` with `customer` also goes to a single shard. This is also referred to as a local join:
+
+```text
+mysql> select c.uname, o.oname, o.product_id from customer c join corder o on c.customer_id = o.customer_id where c.customer_id=1;
++-------+-------+------------+
+| uname | oname | product_id |
++-------+-------+------------+
+| alice | gift | 1 |
+| alice | gift | 2 |
++-------+-------+------------+
+2 rows in set (0.01 sec)
+```
+
+Performing the join without a `customer_id` constraint still results in a local join, but the query is scattered across all shards:
+
+```text
+mysql> select c.uname, o.oname, o.product_id from customer c join corder o on c.customer_id = o.customer_id;
++---------+----------+------------+
+| uname | oname | product_id |
++---------+----------+------------+
+| alice | gift | 1 |
+| alice | gift | 2 |
+| bob | work | 1 |
+| charlie | personal | 2 |
+| dan | personal | 1 |
++---------+----------+------------+
+5 rows in set (0.00 sec)
+```
+
+However, adding a join with `product` results in a cross-shard join for the product part ot the query:
+
+```text
+mysql> select c.uname, o.oname, p.pname from customer c join corder o on c.customer_id = o.customer_id join product p on o.product_id = p.product_id;
++---------+----------+----------+
+| uname | oname | pname |
++---------+----------+----------+
+| alice | gift | monitor |
+| alice | gift | keyboard |
+| bob | work | monitor |
+| charlie | personal | keyboard |
+| dan | personal | monitor |
++---------+----------+----------+
+5 rows in set (0.01 sec)
+```
+
+Although the underlying work performed by Vitess is not visible here, you can see it in the bottom right panel if using the demo app. Alternatively, you can also stream this information with the following command:
+
+```text
+curl localhost:12345/debug/querylog
+[verbose output not shown]
+```
+
+### Foreign Keys
+
+More generically stated: If a table has a foreign key into another table, then Vitess can ensure that the related rows live in the same shard by making them share a common Unique Vindex.
+
+In cases where you choose to group rows based on their foreign key relationships, you have the option to enforce those constraints within each shard at the MySQL level. You can also configure cascade deletes as needed. However, overuse of foreign key constraints is generally discouraged in MySQL.
+
+Foreign key constraints across shards or keyspaces are not supported in Vitess. For example, you cannot specify a foreign key between `corder.product_id` and `product.product_id`.
+
+### Many-to-Many relationships
+
+In the case where a table has relationships with multiple other tables, you can only choose one of those relationships for shard grouping. All other relationships will end up being cross-shard, and will incur cross-shard penalties.
+
+If a table has strong relationships with multiple other tables, and if performance becomes a challenge choosing either way, you can explore the [VReplication Materialization](../../../reference/vreplication/materialize) feature that allows you to materialize a table both ways.
+
+### Enforcing Uniqueness
+
+To enforce global uniqueness for a row in a sharded table, you have to have:
+* A Unique Vindex on the column
+* A Unique constraint at the MySQL level
+
+A Primary Vindex coupled with a Primary Key constraint makes a row globally unique.
+
+A Unique Vindex can also be specified for a non-unique column. In such cases, it is likely that you will be using that column in a where clause, and will require a secondary non-unique index on it at the MySQL level.
diff --git a/content/en/docs/user-guides/vschema-guide/unique-lookup.md b/content/en/docs/user-guides/vschema-guide/unique-lookup.md
new file mode 100644
index 000000000..63bcf1acd
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/unique-lookup.md
@@ -0,0 +1,166 @@
+---
+title: Unique Lookup Vindexes
+weight: 8
+---
+
+Certain application features may require you to point-select orders by their id with a query like this:
+
+```sql
+select * from corder where corder_id=1;
+```
+
+However, issuing this query to Vitess will cause it to scatter this query across all shards because there is no way to know which shard contains that order id. This would be inefficient if the QPS of this query or the number of shards is too high.
+
+Vitess supports the concept of lookup vindexes, also known as cross-shard indexes. You can instruct Vitess to create and manage a lookup vindex for the `corder_id` column. Such a vindex needs to maintain a mapping from `corder_id` to the `keyspace_id` of the row, which will be stored in a lookup table.
+
+This lookup table can be created in any keyspace, and it may or may not be sharded. In this particular case, we are going to create the table in the unsharded product keyspace even though the lookup vindex itself is going to be in the `customer` keyspace:
+
+```sql
+create table corder_keyspace_idx(corder_id bigint not null auto_increment, keyspace_id varbinary(10), primary key(corder_id));
+```
+
+The primary key is `corder_id`. The unique constraint on `corder_id` makes the Lookup Vindex unique: for a given `corder_id` as input, at most one `keyspace_id` can be produced. It is not necessary to name the column as `corder_id`, but it is less confusing to do so.
+
+Since the table is not sharded, we have a trivial VSchema addition:
+
+```json
+ "corder_keyspace_idx": {}
+```
+
+We can now instantiate the Lookup Vindex in the VSchema of the `customer` keyspace:
+
+```json
+ "corder_keyspace_idx": {
+ "type": "consistent_lookup_unique",
+ "params": {
+ "table": "product.corder_keyspace_idx",
+ "from": "corder_id",
+ "to": "keyspace_id"
+ },
+ "owner": "corder"
+ }
+```
+
+* The vindex is given a distinctive name `corder_keyspace_idx` because of its specific input parameters.
+* The vindex type is `consistent_lookup_unique`. We expect this lookup vindex to yield at most one keyspace id for a given input. The `consistent` qualifier is explained below.
+* The `params` section of a Vindex is a set of key-value strings. Each vindex expects a different set of parameters depending on the implementation. A lookup vindex requires the following three parameters:
+ * `table` should be the name of the lookup table. It is recommended that it is fully qualified.
+ * The `from` and `to` fields must reference the column names of the lookup table.
+* The `owner` field indicates that `corder` is responsible for populating the lookup table and keeping it up-to-date. This means that an insert into `corder` will result in a corresponding lookup row being inserted in the lookup table, etc. Lookup vindexes can also be shared, but they can have only one owner each. We will later see an example about how to share lookup vindexes.
+
+{{< info >}}
+Since `corder_keyspace_idx` and `corder` are in different keyspaces, any change that affects the lookup column results in a distributed transaction between the `customer` shard and the `product` keyspace. Usually, a two-phase commit (2PC) protocol would need to be used for the distributed transaction. However, the `consistent` lookup vindex utilizes a special algorithm that orders the commits in such a way that a commit failure resulting in partial commits does not result in inconsistent data. This avoids the extra overheads associated with 2PC.
+{{< /info >}}
+
+Finally, we must associate `customer.corder_id` with the lookup vindex:
+
+```json
+ "column_vindexes": [{
+ "column": "customer_id",
+ "name": "hash"
+ }, {
+ "column": "corder_id",
+ "name": "corder_keyspace_idx"
+ }]
+```
+
+Note that `corder_id` comes after `customer_id` implying that `customer_id` is the Primary Vindex for this table.
+
+Alternate VSchema DDL:
+
+```sql
+alter vschema add table product.corder_keyspace_idx;
+alter vschema on customer.corder add vindex corder_keyspace_idx(corder_id) using consistent_lookup_unique with owner=`corder`, table=`product.corder_keyspace_idx`, from=`corder_id`, to=`keyspace_id`;
+```
+
+{{< info >}}
+An owned lookup vindex (even if unique) cannot be a Primary Vindex because it creates an association against a keyspace id after one has been assigned to the row. The job of computing the keyspace id must therefore be performed by a different unique vindex.
+{{< /info >}}
+
+Bringing up the demo application again, you can now see the lookup table being automatically populated when rows are inserted in `corder`:
+
+```text
+mysql> insert into corder(customer_id, product_id, oname) values (1,1,'gift'),(1,2,'gift'),(2,1,'work'),(3,2,'personal'),(4,1,'personal');
+Query OK, 5 rows affected (0.00 sec)
+
+mysql> select corder_id, hex(keyspace_id) from corder_keyspace_idx;
++-----------+------------------+
+| corder_id | hex(keyspace_id) |
++-----------+------------------+
+| 1 | 166B40B44ABA4BD6 |
+| 2 | 166B40B44ABA4BD6 |
+| 3 | 06E7EA22CE92708F |
+| 4 | 4EB190C9A2FA169C |
+| 5 | D2FD8867D50D2DFE |
++-----------+------------------+
+5 rows in set (0.01 sec)
+```
+
+And then, issuing a query like `select * from corder where corder_id=1` results in two single-shard round-trips instead of a full scatter.
+
+### Reversible Vindexes
+
+Looking at the rows in `corder_keyspace_idx` reveals a few things. We get to now see actual keyspace id values that were previously invisible. We can also notice that two different inputs `1` and `2` map to the same keyspace id `166B40B44ABA4BD6`. In other words, a unique vindex does not necessarily guarantee that two different values yield different keyspace ids. In fact, this is derived from the fact that there are two order rows for customer id `1`.
+
+Vindexes that do have a one-to-one correspondence between the input value and keyspace id , like `hash`, are known as reversible vindexes: Given a keyspace id, the input value can be back-computed. This property will be used in a later example.
+
+### Backfill
+
+Creating a lookup vindex after the main table already contains rows does not automatically backfill the lookup table for the existing entries. Only newer inserts cause automatic population of the lookup table. This backfill can be set up using the `CreateLookupVindex` command covered below.
+
+### Checklist
+
+Creating a unique lookup Vindex is an elaborate process. It is good to use the following checklist if this is done manually:
+* Create the lookup table as sharded or unsharded. Make the `from` column the primary key.
+* Create a VSchema entry for the lookup table. If sharded, assign a Primary Vindex for the `from` column.
+* Create the lookup vindex in the VSchema of the sharded keyspace:
+ * Give it a distinct name
+ * Specify the type as `consistent_lookup_unique`
+ * Under `params`: specify the properties of the lookup table
+ * Specify the `Owner` as the main table
+* Associate the column of the owner table with the new Vindex.
+
+### CreateLookupVindex command
+
+vtctld supports the [CreateLookupVindex](../../configuration-advanced/createlookupvindex) command that can perform all the above steps as well as the backfill.
+
+{{< warning >}}
+This will not work against the `vtcombo` based demo app because it does not support vreplication. You can only try this against a real Vitess cluster.
+{{< /warning >}}
+
+Save the following json into a file, say `corder_keyspace_idx.json`:
+
+```json
+{
+ "sharded": true,
+ "vindexes": {
+ "corder_keyspace_idx": {
+ "type": "consistent_lookup_unique",
+ "params": {
+ "table": "product.corder_keyspace_idx",
+ "from": "corder_id",
+ "to": "keyspace_id"
+ },
+ "owner": "corder"
+ }
+ },
+ "tables": {
+ "corder": {
+ "column_vindexes": [{
+ "column": "corder_id",
+ "name": "corder_keyspace_idx"
+ }],
+ }
+ }
+}
+```
+
+And issue the vtctlclient command:
+
+```sh
+$ vtctlclient -server CreateLookupVindex -tablet_types=REPLICA customer "$(cat corder_keyspace_idx.json)"
+```
+
+The workflow automatically infers the schema and vschema for the lookup table and creates it. It also sets up the necessary VReplication streams to backfill the lookup table.
+
+After the backfill is done, you should clean up the workflow. More detailed instructions are available in the [CreateLookupVindex Reference](../../configuration-advanced/createlookupvindex)
diff --git a/content/en/docs/user-guides/vschema-guide/unsharded.md b/content/en/docs/user-guides/vschema-guide/unsharded.md
new file mode 100644
index 000000000..299c51bb4
--- /dev/null
+++ b/content/en/docs/user-guides/vschema-guide/unsharded.md
@@ -0,0 +1,85 @@
+---
+title: Unsharded Keyspace
+weight: 4
+---
+
+We are going to start with configuring the `product` table in the unsharded keyspace `product`. The schema file should be as follows:
+
+```sql
+create table product(product_id bigint auto_increment, pname varchar(128), primary key(product_id));
+```
+
+`product_id` is the primary key for product, and it is also configured to use MySQL’s `auto_increment` feature that allows you to automatically generate unique values for it.
+
+We also need to create a VSchema for the `product` keyspace and specify that `product` is a table in the keyspace:
+
+```json
+{
+ "sharded": false,
+ "tables": {
+ "product": {}
+ }
+}
+```
+
+The json states that the keyspace is not sharded. The product table is specified in the “tables” section of the json. This is because there are other sections that we will introduce later.
+
+For unsharded keyspaces, no additional metadata is needed for regular tables. So, their entry is empty.
+
+Alternate VSchema DDL:
+
+```sql
+alter vschema add table product.product;
+```
+
+{{< info >}}
+If `product` is the only keyspace in the cluster, a vschema is unnecessary. Vitess treats single keyspace clusters as a special case and optimistically forwards all queries to that keyspace even if there is no table metadata present in the vschema. But it is best practice to provide a full vschema to avoid future complications.
+{{< /info >}}
+
+Bringing up the cluster will allow you to access the `product` table. You can now insert rows into the table:
+
+```text
+$ mysql -h 127.0.0.1 -P 12348
+[snip]
+mysql> insert into product(pname) values ('monitor'), ('keyboard');
+Query OK, 2 rows affected (0.00 sec)
+
+mysql> select * from product;
++------------+----------+
+| product_id | pname |
++------------+----------+
+| 1 | monitor |
+| 2 | keyboard |
++------------+----------+
+2 rows in set (0.00 sec)
+```
+The insert does not specify values for `product_id`, because we are relying on MySQL’s `auto_increment` feature to populate it.
+
+You will notice that we did not connect to the `product` database or issue a `use` statement to select it. This is the ‘unspecified’ mode supported by Vitess. As long as a table name can be uniquely identified from the vschemas, Vitess will automatically direct the query to the correct keyspace.
+
+You can also connect or specify keyspaces as if they were MySQL databases. The following constructs are valid:
+
+```text
+mysql> select * from product.product;
++------------+----------+
+| product_id | pname |
++------------+----------+
+| 1 | monitor |
+| 2 | keyboard |
++------------+----------+
+2 rows in set (0.00 sec)
+
+mysql> use product;
+Reading table information for completion of table and column names
+You can turn off this feature to get a quicker startup with -A
+
+Database changed
+mysql> select * from product;
++------------+----------+
+| product_id | pname |
++------------+----------+
+| 1 | monitor |
+| 2 | keyboard |
++------------+----------+
+2 rows in set (0.01 sec)
+```
diff --git a/content/zh/docs/_index.md b/content/zh/docs/_index.md
index c58c10d02..125b7cc02 100644
--- a/content/zh/docs/_index.md
+++ b/content/zh/docs/_index.md
@@ -1,6 +1,6 @@
---
title: Vitess 文档
-description: 你想了解的有关世界上最具扩展性的开源MySQL平台的一切，都在这里
+description: 因为这些文档不维护，所以它们是旧的。你想了解的有关世界上最具扩展性的开源MySQL平台的一切，都在这里
notoc: true
---
diff --git a/content/zh/docs/concepts/_index.md b/content/zh/docs/concepts/_index.md
index d5b1abb46..8f294d71e 100644
--- a/content/zh/docs/concepts/_index.md
+++ b/content/zh/docs/concepts/_index.md
@@ -3,3 +3,6 @@ title: 概念
description: 学习Vitess核心的概念和术语
weight: 3
---
+{{< info >}}
+因为这些文档不维护，所以它们是旧的。
+{{< /info >}}
\ No newline at end of file
diff --git a/content/zh/docs/contributing/_index.md b/content/zh/docs/contributing/_index.md
index 64efc08f5..a51ea1bc6 100644
--- a/content/zh/docs/contributing/_index.md
+++ b/content/zh/docs/contributing/_index.md
@@ -3,6 +3,9 @@ title: 贡献Vitess
description: 我们热爱所有参与贡献的人，这篇文章将描述您如何参与到贡献Vitess中来
weight: 6
---
+{{< info >}}
+因为这些文档不维护，所以它们是旧的。
+{{< /info >}}
什么，听说您想对vitess作出贡献？这简直太棒了！
diff --git a/content/zh/docs/contributing/build-from-source.md b/content/zh/docs/contributing/build-from-source.md
index 8b273408e..bc31684f2 100644
--- a/content/zh/docs/contributing/build-from-source.md
+++ b/content/zh/docs/contributing/build-from-source.md
@@ -23,7 +23,7 @@ featured: true
Vitess依赖如下软件和库:
-1. [Install Go 1.11+](http://golang.org/doc/install).
+1. [Install Go 1.15+](http://golang.org/doc/install).
2. Install MySQL:
```bash
diff --git a/content/zh/docs/faq/_index.md b/content/zh/docs/faq/_index.md
index a7b37194e..a67c01ec5 100644
--- a/content/zh/docs/faq/_index.md
+++ b/content/zh/docs/faq/_index.md
@@ -4,6 +4,9 @@ description: Frequently Asked Questions about Vitess
weight: 8
featured: true
---
+{{< info >}}
+因为这些文档不维护，所以它们是旧的。
+{{< /info >}}
## Does the application need to know about the sharding scheme underneath Vitess?
diff --git a/content/zh/docs/get-started/_index.md b/content/zh/docs/get-started/_index.md
index ccc0f559d..a1dfb361d 100644
--- a/content/zh/docs/get-started/_index.md
+++ b/content/zh/docs/get-started/_index.md
@@ -3,5 +3,8 @@ title: 入门
description: 在你最喜爱的平台上部署Vitess
weight: 2
---
+{{< info >}}
+因为这些文档不维护，所以它们是旧的。
+{{< /info >}}
Vitess支持在以上平台上进行二进制部署。 如果你对如何构建二进制文件很感兴趣，或者想对vitess做贡献。可以参考[从源码编译](../contributing/build-from-source)。
diff --git a/content/zh/docs/launching/_index.md b/content/zh/docs/launching/_index.md
index fb961adc5..ca01bd58d 100644
--- a/content/zh/docs/launching/_index.md
+++ b/content/zh/docs/launching/_index.md
@@ -3,5 +3,8 @@ title: Launching
description: Everything you need to know to launch your project with Vitess
weight: 7
---
+{{< info >}}
+因为这些文档不维护，所以它们是旧的。
+{{< /info >}}
This section seeks to outline everything you need to launch your project successfully with Vitess. It will be updated with additional documentation.
diff --git a/content/zh/docs/launching/server-configuration.md b/content/zh/docs/launching/server-configuration.md
index 7748566bf..ee9a03abe 100644
--- a/content/zh/docs/launching/server-configuration.md
+++ b/content/zh/docs/launching/server-configuration.md
@@ -412,7 +412,7 @@ This URL prints out a simple "ok" or “not ok” string that can be used to che
#### /querylogz, /debug/querylog, /txlogz, /debug/txlog
-* /debug/querylog is a never-ending stream of currently executing queries with verbose information about each query. This URL can generate a lot of data because it streams every query processed by VTTablet. The details are as per this function: https://github.com/vitessio/vitess/blob/master/go/vt/tabletserver/logstats.go#L202
+* /debug/querylog is a never-ending stream of currently executing queries with verbose information about each query. This URL can generate a lot of data because it streams every query processed by VTTablet. The details are as per this function: https://github.com/vitessio/vitess/blob/master/go/vt/vttablet/tabletserver/tabletenv/logstats.go#L202
* /querylogz is a limited human readable version of /debug/querylog. It prints the next 300 queries by default. The limit can be specified with a limit=N parameter on the URL.
* /txlogz is like /querylogz, but for transactions.
* /debug/txlog is the JSON counterpart to /txlogz.
diff --git a/content/zh/docs/overview/_index.md b/content/zh/docs/overview/_index.md
index 791e5d531..9419c7fd1 100644
--- a/content/zh/docs/overview/_index.md
+++ b/content/zh/docs/overview/_index.md
@@ -3,5 +3,8 @@ title: 概览
description: Vitess 概述
weight: 1
---
+{{< info >}}
+因为这些文档不维护，所以它们是旧的。
+{{< /info >}}
Vitess概述文档提供了有关Vitess的大致信息， 您可以获得更多的详细信息在[Get Started](../get-started) 章节和 [User Guides](../user-guides)章节.
diff --git a/content/zh/docs/reference/_index.md b/content/zh/docs/reference/_index.md
index e514af584..23d0931e1 100644
--- a/content/zh/docs/reference/_index.md
+++ b/content/zh/docs/reference/_index.md
@@ -4,3 +4,6 @@ description: Detailed information about specific Vitess functionality
weight: 5
---
+{{< info >}}
+因为这些文档不维护，所以它们是旧的。
+{{< /info >}}
\ No newline at end of file
diff --git a/content/zh/docs/schema-management/_index.md b/content/zh/docs/schema-management/_index.md
index c9c3aec90..d4b4aa4e4 100644
--- a/content/zh/docs/schema-management/_index.md
+++ b/content/zh/docs/schema-management/_index.md
@@ -3,6 +3,9 @@ title: Schema Management
description: More information about all things schema and VSchema
weight: 6
---
+{{< info >}}
+因为这些文档不维护，所以它们是旧的。
+{{< /info >}}
Using Vitess requires you to work with two different types of schemas:
@@ -87,7 +90,7 @@ Vitess' schema modification functionality is designed the following goals in min
* Guarantee very little downtime (or no downtime) for most schema updates.
* Do not store permanent schema data in the topology server.
-Note that, at this time, Vitess only supports [data definition statements](https://dev.mysql.com/doc/refman/5.6/en/sql-syntax-data-definition.html) that create, modify, or delete database tables. For instance, `ApplySchema` does not affect stored procedures or grants.
+Note that, at this time, Vitess only supports [data definition statements](https://dev.mysql.com/doc/refman/5.6/en/sql-data-definition-statements.html) that create, modify, or delete database tables. For instance, `ApplySchema` does not affect stored procedures or grants.
The [ApplySchema](../reference/vtctl/#applyvschema) command applies a schema change to the specified keyspace on every master tablet, running in parallel on all shards. Changes are then propagated to slaves via replication. The command format is: `ApplySchema {-sql= || -sql_file=} `
diff --git a/content/zh/docs/user-guides/_index.md b/content/zh/docs/user-guides/_index.md
index 7d07152fe..e37eddc60 100644
--- a/content/zh/docs/user-guides/_index.md
+++ b/content/zh/docs/user-guides/_index.md
@@ -3,6 +3,9 @@ title: User Guides
description: Practical Guides for Most Scenarios
weight: 3
---
+{{< info >}}
+因为这些文档不维护，所以它们是旧的。
+{{< /info >}}
## Platform support
@@ -37,7 +40,7 @@ validating its consistency across tablets in a shard or across all
shards in a keyspace.
In addition, Vitess supports
-[data definition statements](https://dev.mysql.com/doc/refman/5.6/en/sql-syntax-data-definition.html)
+[data definition statements](https://dev.mysql.com/doc/refman/5.6/en/sql-data-definition-statements.html)
that create, modify, or delete database tables. Vitess executes
schema changes on the master tablet within each shard, and those
changes then propagate to slave tablets via replication. Vitess does
diff --git a/i18n/en.toml b/i18n/en.toml
index 34c75066a..ce2133470 100644
--- a/i18n/en.toml
+++ b/i18n/en.toml
@@ -55,4 +55,4 @@ other = "Slack"
other = "Stack Overflow"
[search]
-other = "Search"
+other = "Search the docs"
diff --git a/i18n/zh.toml b/i18n/zh.toml
index 56c4bbcda..847aa361e 100644
--- a/i18n/zh.toml
+++ b/i18n/zh.toml
@@ -55,4 +55,4 @@ other = "Slack"
other = "Stack Overflow"
[search]
-other = "Search"
+other = "Search the docs"
diff --git a/layouts/_default/_markup/render-heading.html b/layouts/_default/_markup/render-heading.html
new file mode 100644
index 000000000..116b3b863
--- /dev/null
+++ b/layouts/_default/_markup/render-heading.html
@@ -0,0 +1 @@
+{{ .Text | safeHTML }} #
diff --git a/layouts/_default/baseof.html b/layouts/_default/baseof.html
index 54584c9d9..5464fb0ba 100644
--- a/layouts/_default/baseof.html
+++ b/layouts/_default/baseof.html
@@ -13,7 +13,7 @@
{{ partial "css.html" . }}
-
+
{{ block "main" . }}{{ end }}
diff --git a/layouts/_default/rss.xml b/layouts/_default/rss.xml
new file mode 100644
index 000000000..965c42514
--- /dev/null
+++ b/layouts/_default/rss.xml
@@ -0,0 +1,43 @@
+
+{{- $pctx := . -}}
+{{- if .IsHome -}}{{ $pctx = .Site }}{{- end -}}
+{{- $pages := slice -}}
+{{- if or $.IsHome $.IsSection -}}
+{{- $pages = $pctx.RegularPages -}}
+{{- else -}}
+{{- $pages = $pctx.Pages -}}
+{{- end -}}
+{{- $limit := .Site.Config.Services.RSS.Limit -}}
+{{- if ge $limit 1 -}}
+{{- $pages = $pages | first $limit -}}
+{{- end -}}
+{{- printf "" | safeHTML }}
+
+
+ {{ if eq .Title .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} on {{ end }}{{ .Site.Title }}{{ end }}
+ {{ .Permalink }}
+ Recent content {{ if ne .Title .Site.Title }}{{ with .Title }}in {{.}} {{ end }}{{ end }}on {{ .Site.Title }}
+ Hugo -- gohugo.io{{ with .Site.LanguageCode }}
+ {{.}}{{end}}{{ with .Site.Author.email }}
+ {{.}}{{ with $.Site.Author.name }} ({{.}}){{end}}{{end}}{{ with .Site.Author.email }}
+ {{.}}{{ with $.Site.Author.name }} ({{.}}){{end}}{{end}}{{ with .Site.Copyright }}
+ {{.}}{{end}}{{ if not .Date.IsZero }}
+ {{ .Date.Format "Mon, 02 Jan 2006 15:04:05 -0700" | safeHTML }}{{ end }}
+ {{- with .OutputFormats.Get "RSS" -}}
+ {{ printf "" .Permalink .MediaType | safeHTML }}
+ {{- end -}}
+ {{ range $pages }}
+
+ {{ .Title }}
+ {{ .Permalink }}
+ {{ .Date.Format "Mon, 02 Jan 2006 15:04:05 -0700" | safeHTML }}
+ {{ with .Site.Author.email }}{{.}}{{ with $.Site.Author.name }} ({{.}}){{end}}{{end}}
+ {{ .Permalink }}
+ {{ range .Params.tags }}
+ {{ . }}
+ {{ end }}
+ {{ .Summary | html }}
+
+ {{ end }}
+
+
diff --git a/layouts/_default/sitemap.xml b/layouts/_default/sitemap.xml
new file mode 100644
index 000000000..a94098b02
--- /dev/null
+++ b/layouts/_default/sitemap.xml
@@ -0,0 +1,22 @@
+{{ printf "" | safeHTML }}
+
+ {{ range .Data.Pages }}
+
+ {{ .Permalink }}{{ if not .Lastmod.IsZero }}
+ {{ safeHTML ( .Lastmod.Format "2006-01-02T15:04:05-07:00" ) }}{{ end }}{{ with .Sitemap.ChangeFreq }}
+ {{ . }}{{ end }}{{ if ge .Sitemap.Priority 0.0 }}
+ {{ .Sitemap.Priority }}{{ end }}{{ if .IsTranslated }}{{ range .Translations }}
+ {{ end }}
+ {{ end }}
+
+ {{ end }}
+
diff --git a/layouts/docs/section.html b/layouts/docs/section.html
index b300c1412..c4a163135 100644
--- a/layouts/docs/section.html
+++ b/layouts/docs/section.html
@@ -10,7 +10,7 @@
+
+{{/*
+ The "navlist" template recursively renders nested lists to any depth.
+ It is called with two parameters, "currentRelPermalink" and "entries",
+ described inline below.
+
+ The structure of the sidebar matches the structure of `content/docs` (for English)
+ or `content/xx/docs` (for other languages). This means that sidebar structure and
+ content will vary between languages.
+*/}}
+{{ define "navlist" }}
+ {{/* `currentRelPermalink` is the RelPermalink of the current content page. */}}
+ {{ $currentRelPermalink := .currentRelPermalink}}
+ {{/* `entries` is a list of Pages */}}
+ {{ $entries := .entries }}
+
+
+ 
+ 
+ 
+ 
+ 
+ 
-
-