Experimental: automated, scheduled, dependency free online DDL via gh-ost/pt-online-schema-change

.github/workflows/cluster_endtoend.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
+        name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 26]
 
     steps:
     - name: Set up Go

Makefile

@@ -54,7 +54,8 @@ ifndef NOBANNER
 	echo $$(date): Building source tree
 endif
 	bash ./build.env
-	go install $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/...
+	go install $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/... && \
+		(cd go/cmd/vttablet && go run github.com/GeertJohan/go.rice/rice append --exec=../../../bin/vttablet)
 
 debug:
 ifndef NOBANNER

docker/mini/vttablet-mini-up.sh

@@ -50,8 +50,6 @@ vttablet \
  -init_shard $shard \
  -init_tablet_type $tablet_type \
  -health_check_interval 5s \
- -heartbeat_enable \
- -heartbeat_interval 250ms \
  -enable_semi_sync \
  -enable_replication_reporter \
  -backup_storage_implementation file \

examples/local/scripts/vttablet-up.sh

@@ -47,8 +47,6 @@ vttablet \
  -health_check_interval 5s \
  -enable_semi_sync \
  -enable_replication_reporter \
- -heartbeat_enable \
- -heartbeat_interval 250ms \
  -backup_storage_implementation file \
  -file_backup_storage_root $VTDATAROOT/backups \
  -restore_from_backup \

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/Azure/go-autorest/autorest v0.10.0 // indirect
 	github.com/GeertJohan/go.rice v1.0.0
 	github.com/PuerkitoBio/goquery v1.5.1
-	github.com/TylerBrock/colorjson v0.0.0-20180527164720-95ec53f28296
 	github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878
 	github.com/aws/aws-sdk-go v1.28.8
@@ -25,13 +24,14 @@ require (
 	github.com/evanphx/json-patch v4.5.0+incompatible
 	github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab
 	github.com/go-sql-driver/mysql v1.5.0
-	github.com/gogo/protobuf v1.3.1
+	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
 	github.com/golang/mock v1.3.1
 	github.com/golang/protobuf v1.3.2
 	github.com/golang/snappy v0.0.1
 	github.com/google/go-cmp v0.4.0
+	github.com/google/uuid v1.1.1
 	github.com/googleapis/gnostic v0.2.0 // indirect
 	github.com/gorilla/websocket v1.4.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
@@ -52,13 +52,11 @@ require (
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/krishicks/yaml-patch v0.0.10
 	github.com/magiconair/properties v1.8.1
-	github.com/manifoldco/promptui v0.7.0
 	github.com/martini-contrib/auth v0.0.0-20150219114609-fa62c19b7ae8
 	github.com/martini-contrib/gzip v0.0.0-20151124214156-6c035326b43f
 	github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11
 	github.com/mattn/go-sqlite3 v1.14.0
 	github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1
-	github.com/mitchellh/go-ps v1.0.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.0 // indirect
 	github.com/mitchellh/mapstructure v1.2.3 // indirect
 	github.com/montanaflynn/stats v0.6.3
@@ -80,7 +78,6 @@ require (
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/sjmudd/stopwatch v0.0.0-20170613150411-f380bf8a9be1
 	github.com/smartystreets/goconvey v1.6.4 // indirect
-	github.com/spf13/cobra v0.0.5
 	github.com/stretchr/testify v1.4.0
 	github.com/tchap/go-patricia v0.0.0-20160729071656-dd168db6051b
 	github.com/tebeka/selenium v0.9.9

go/cmd/vttablet/vttablet.go

@@ -32,11 +32,14 @@ import (
 	"vitess.io/vitess/go/vt/tableacl/simpleacl"
 	"vitess.io/vitess/go/vt/topo"
 	"vitess.io/vitess/go/vt/topo/topoproto"
+	"vitess.io/vitess/go/vt/vttablet/onlineddl"
 	"vitess.io/vitess/go/vt/vttablet/tabletmanager"
 	"vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication"
 	"vitess.io/vitess/go/vt/vttablet/tabletserver"
 	"vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv"
 	"vitess.io/vitess/go/yaml2"
+
+	rice "github.com/GeertJohan/go.rice"
 )
 
 var (
@@ -77,6 +80,10 @@ func main() {
 	mysqld := mysqlctl.NewMysqld(config.DB)
 	servenv.OnClose(mysqld.Close)
 
+	if err := extractOnlineDDL(); err != nil {
+		log.Exitf("failed to extract online DDL binaries: %v", err)
+	}
+
 	// Initialize and start tm.
 	gRPCPort := int32(0)
 	if servenv.GRPCPort != nil {
@@ -157,6 +164,27 @@ func initConfig(tabletAlias *topodatapb.TabletAlias) (*tabletenv.TabletConfig, *
 	return config, mycnf
 }
 
+// extractOnlineDDL extracts the gh-ost binary from this executable. gh-ost is appended
+// to vttablet executable by `make build` and via ricebox
+func extractOnlineDDL() error {
+	riceBox, err := rice.FindBox("../../../resources/bin")
+	if err != nil {
+		return err
+	}
+
+	if binaryFileName, isOverride := onlineddl.GhostBinaryFileName(); !isOverride {
+		ghostBinary, err := riceBox.Bytes("gh-ost")
+		if err != nil {
+			return err
+		}
+		if err := ioutil.WriteFile(binaryFileName, ghostBinary, 0755); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func createTabletServer(config *tabletenv.TabletConfig, ts *topo.Server, tabletAlias *topodatapb.TabletAlias) *tabletserver.TabletServer {
 	if *tableACLConfig != "" {
 		// To override default simpleacl, other ACL plugins must set themselves to be default ACL factory

go/test/endtoend/cluster/vtctlclient_process.go

@@ -47,14 +47,20 @@ func (vtctlclient *VtctlClientProcess) InitShardMaster(Keyspace string, Shard st
 	return err
 }
 
-// ApplySchema applies SQL schema to the keyspace
-func (vtctlclient *VtctlClientProcess) ApplySchema(Keyspace string, SQL string) (err error) {
-	return vtctlclient.ExecuteCommand(
+// ApplySchemaWithOutput applies SQL schema to the keyspace
+func (vtctlclient *VtctlClientProcess) ApplySchemaWithOutput(Keyspace string, SQL string) (result string, err error) {
+	return vtctlclient.ExecuteCommandWithOutput(
 		"ApplySchema",
 		"-sql", SQL,
 		Keyspace)
 }
 
+// ApplySchema applies SQL schema to the keyspace
+func (vtctlclient *VtctlClientProcess) ApplySchema(Keyspace string, SQL string) (err error) {
+	_, err = vtctlclient.ApplySchemaWithOutput(Keyspace, SQL)
+	return err
+}
+
 // ApplyVSchema applies vitess schema (JSON format) to the keyspace
 func (vtctlclient *VtctlClientProcess) ApplyVSchema(Keyspace string, JSON string) (err error) {
 	return vtctlclient.ExecuteCommand(
@@ -64,6 +70,36 @@ func (vtctlclient *VtctlClientProcess) ApplyVSchema(Keyspace string, JSON string
 	)
 }
 
+// OnlineDDLShowRecent responds with recent schema migration list
+func (vtctlclient *VtctlClientProcess) OnlineDDLShowRecent(Keyspace string) (result string, err error) {
+	return vtctlclient.ExecuteCommandWithOutput(
+		"OnlineDDL",
+		Keyspace,
+		"show",
+		"recent",
+	)
+}
+
+// OnlineDDLCancelMigration cancels a given migration uuid
+func (vtctlclient *VtctlClientProcess) OnlineDDLCancelMigration(Keyspace, uuid string) (result string, err error) {
+	return vtctlclient.ExecuteCommandWithOutput(
+		"OnlineDDL",
+		Keyspace,
+		"cancel",
+		uuid,
+	)
+}
+
+// OnlineDDLRetryMigration retries a given migration uuid
+func (vtctlclient *VtctlClientProcess) OnlineDDLRetryMigration(Keyspace, uuid string) (result string, err error) {
+	return vtctlclient.ExecuteCommandWithOutput(
+		"OnlineDDL",
+		Keyspace,
+		"retry",
+		uuid,
+	)
+}
+
 // ExecuteCommand executes any vtctlclient command
 func (vtctlclient *VtctlClientProcess) ExecuteCommand(args ...string) (err error) {
 	output, err := vtctlclient.ExecuteCommandWithOutput(args...)

go/test/endtoend/cluster/vtctld_process.go

@@ -67,6 +67,8 @@ func (vtctld *VtctldProcess) Setup(cell string, extraArgs ...string) (err error)
 		// hard-code these two soon-to-be deprecated drain values.
 		"-wait_for_drain_sleep_rdonly", "1s",
 		"-wait_for_drain_sleep_replica", "1s",
+		// short online-ddl check interval to hasten tests
+		"-online_ddl_check_interval", "2s",
 		"-log_dir", vtctld.LogDir,
 		"-port", fmt.Sprintf("%d", vtctld.Port),
 		"-grpc_port", fmt.Sprintf("%d", vtctld.GrpcPort),