WIP: ApplySchema to run online schema changes via gh-ost

docker/local/install_local_dependencies.sh

@@ -18,5 +18,10 @@ rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
 
 mkdir -p /var/run/etcd && chown -R vitess:vitess /var/run/etcd
 
+# Install gh-ost
+curl -k -L https://github.com/openark/gh-ost/releases/download/v1.1.0/gh-ost-binary-linux-20200805092717.tar.gz -o /tmp/gh-ost.tar.gz
+(cd /tmp/ && tar xzf gh-ost.tar.gz)
+cp /tmp/gh-ost /usr/bin
+
 # Clean up files we won't need in the final image.
 rm -rf /var/lib/apt/lists/*

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/golang/protobuf v1.3.2
 	github.com/golang/snappy v0.0.1
 	github.com/google/go-cmp v0.4.0
+	github.com/google/uuid v1.1.1
 	github.com/googleapis/gnostic v0.2.0 // indirect
 	github.com/gorilla/websocket v1.4.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
@@ -41,11 +42,12 @@ require (
 	github.com/hashicorp/serf v0.9.2 // indirect
 	github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428
 	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/julienschmidt/httprouter v1.3.0
 	github.com/klauspost/compress v1.4.1 // indirect
 	github.com/klauspost/cpuid v1.2.0 // indirect
 	github.com/klauspost/pgzip v1.2.4
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/krishicks/yaml-patch v0.0.10
+	github.com/krishicks/yaml-patch v0.0.10 // indirect
 	github.com/magiconair/properties v1.8.1
 	github.com/manifoldco/promptui v0.7.0
 	github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1

go.sum

@@ -363,6 +363,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU=
 github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
@@ -685,6 +687,7 @@ golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ym
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=

go/sqltypes/named_result.go

@@ -0,0 +1,140 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sqltypes
+
+import (
+	"errors"
+
+	querypb "vitess.io/vitess/go/vt/proto/query"
+)
+
+var (
+	// ErrNoSuchField indicates a search for a value by an unknown field/column name
+	ErrNoSuchField = errors.New("No such field in RowNamedValues")
+)
+
+// RowNamedValues contains a row's values as a map based on Field (aka table column) name
+type RowNamedValues map[string]Value
+
+// ToString returns the named field as string
+func (r RowNamedValues) ToString(fieldName string) (string, error) {
+	if v, ok := r[fieldName]; ok {
+		return v.ToString(), nil
+	}
+	return "", ErrNoSuchField
+}
+
+// AsString returns the named field as string, or default value if nonexistent/error
+func (r RowNamedValues) AsString(fieldName string, def string) string {
+	if v, err := r.ToString(fieldName); err == nil {
+		return v
+	}
+	return def
+}
+
+// ToInt64 returns the named field as int64
+func (r RowNamedValues) ToInt64(fieldName string) (int64, error) {
+	if v, ok := r[fieldName]; ok {
+		return v.ToInt64()
+	}
+	return 0, ErrNoSuchField
+}
+
+// AsInt64 returns the named field as int64, or default value if nonexistent/error
+func (r RowNamedValues) AsInt64(fieldName string, def int64) int64 {
+	if v, err := r.ToInt64(fieldName); err == nil {
+		return v
+	}
+	return def
+}
+
+// ToUint64 returns the named field as uint64
+func (r RowNamedValues) ToUint64(fieldName string) (uint64, error) {
+	if v, ok := r[fieldName]; ok {
+		return v.ToUint64()
+	}
+	return 0, ErrNoSuchField
+}
+
+// AsUint64 returns the named field as uint64, or default value if nonexistent/error
+func (r RowNamedValues) AsUint64(fieldName string, def uint64) uint64 {
+	if v, err := r.ToUint64(fieldName); err == nil {
+		return v
+	}
+	return def
+}
+
+// ToBool returns the named field as bool
+func (r RowNamedValues) ToBool(fieldName string) (bool, error) {
+	if v, ok := r[fieldName]; ok {
+		return v.ToBool()
+	}
+	return false, ErrNoSuchField
+}
+
+// AsBool returns the named field as bool, or default value if nonexistent/error
+func (r RowNamedValues) AsBool(fieldName string, def bool) bool {
+	if v, err := r.ToBool(fieldName); err == nil {
+		return v
+	}
+	return def
+}
+
+// NamedResult represents a query result with named values as opposed to ordinal values.
+type NamedResult struct {
+	Fields       []*querypb.Field `json:"fields"`
+	RowsAffected uint64           `json:"rows_affected"`
+	InsertID     uint64           `json:"insert_id"`
+	Rows         []RowNamedValues `json:"rows"`
+}
+
+// ToNamedResult converts a Result struct into a new NamedResult struct
+func ToNamedResult(result *Result) (r *NamedResult) {
+	if result == nil {
+		return r
+	}
+	r = &NamedResult{
+		Fields:       result.Fields,
+		RowsAffected: result.RowsAffected,
+		InsertID:     result.InsertID,
+	}
+	columnOrdinals := make(map[int]string)
+	for i, field := range result.Fields {
+		columnOrdinals[i] = field.Name
+	}
+	r.Rows = make([]RowNamedValues, len(result.Rows))
+	for rowIndex, row := range result.Rows {
+		namedRow := make(RowNamedValues)
+		for i, value := range row {
+			namedRow[columnOrdinals[i]] = value
+		}
+		r.Rows[rowIndex] = namedRow
+	}
+	return r
+}
+
+// Row assumes this result has exactly one row, and returns it, or else returns nil.
+// It is useful for queries like:
+// - select count(*) from ...
+// - select @@read_only
+// - select UNIX_TIMESTAMP() from dual
+func (r *NamedResult) Row() RowNamedValues {
+	if len(r.Rows) != 1 {
+		return nil
+	}
+	return r.Rows[0]
+}

go/sqltypes/named_result_test.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sqltypes
+
+import (
+	"reflect"
+	"testing"
+
+	querypb "vitess.io/vitess/go/vt/proto/query"
+)
+
+func TestToNamedResult(t *testing.T) {
+	in := &Result{
+		Fields: []*querypb.Field{{
+			Name: "id",
+			Type: Int64,
+		}, {
+			Name: "status",
+			Type: VarChar,
+		}},
+		InsertID:     1,
+		RowsAffected: 2,
+		Rows: [][]Value{
+			{TestValue(Int64, "1"), TestValue(VarChar, "first")},
+			{TestValue(Int64, "2"), TestValue(VarChar, "second")},
+			{TestValue(Int64, "3"), TestValue(VarChar, "third")},
+		},
+	}
+	named := in.Named()
+	for i := range in.Rows {
+		{
+			want := in.Rows[i][0]
+			got := named.Rows[i]["id"]
+			if !reflect.DeepEqual(want, got) {
+				t.Errorf("Named:%+v\n, want:%+v\n", got, want)
+			}
+		}
+		{
+			want := in.Rows[i][1]
+			got := named.Rows[i]["status"]
+			if !reflect.DeepEqual(want, got) {
+				t.Errorf("Named:%+v\n, want:%+v\n", got, want)
+			}
+		}
+	}
+}

go/sqltypes/result.go

@@ -218,3 +218,8 @@ func (result *Result) AppendResult(src *Result) {
 	}
 	result.Rows = append(result.Rows, src.Rows...)
 }
+
+// Named returns a NamedResult based on this struct
+func (result *Result) Named() *NamedResult {
+	return ToNamedResult(result)
+}

go/sqltypes/value.go

@@ -20,6 +20,7 @@ package sqltypes
 import (
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 
@@ -37,6 +38,9 @@ var (
 	DontEscape = byte(255)
 
 	nullstr = []byte("null")
+
+	// ErrIncompatibleTypeCast indicates a casting problem
+	ErrIncompatibleTypeCast = errors.New("Cannot convert value to desired type")
 )
 
 // BinWriter interface is used for encoding values.
@@ -203,6 +207,39 @@ func (v Value) Len() int {
 	return len(v.val)
 }
 
+// ToInt64 returns the value as MySQL would return it as a int64.
+func (v Value) ToInt64() (int64, error) {
+	if !v.IsIntegral() {
+		return 0, ErrIncompatibleTypeCast
+	}
+
+	return strconv.ParseInt(v.ToString(), 10, 64)
+}
+
+// ToUint64 returns the value as MySQL would return it as a uint64.
+func (v Value) ToUint64() (uint64, error) {
+	if !v.IsIntegral() {
+		return 0, ErrIncompatibleTypeCast
+	}
+
+	return strconv.ParseUint(v.ToString(), 10, 64)
+}
+
+// ToBool returns the value as a bool value
+func (v Value) ToBool() (bool, error) {
+	i, err := v.ToInt64()
+	if err != nil {
+		return false, err
+	}
+	switch i {
+	case 0:
+		return false, nil
+	case 1:
+		return true, nil
+	}
+	return false, ErrIncompatibleTypeCast
+}
+
 // ToString returns the value as MySQL would return it as string.
 // If the value is not convertible like in the case of Expression, it returns nil.
 func (v Value) ToString() string {

go/test/endtoend/tabletmanager/tablet_health_test.go

@@ -126,7 +126,7 @@ func TestHealthCheck(t *testing.T) {
 
 	// stop replication, make sure we don't go unhealthy.
 	// TODO: replace with StopReplication once StopSlave has been removed
-	err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopSlave", rTablet.Alias)
+	err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", rTablet.Alias)
 	require.NoError(t, err)
 	err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias)
 	require.NoError(t, err)