Add initial support for capabilities

config/mycnf/default.cnf

@@ -2,7 +2,7 @@
 
 sql_mode = STRICT_TRANS_TABLES
 back_log = 50
-binlog_format = statement
+binlog_format = row
 character_set_server = utf8
 collation_server = utf8_general_ci
 connect_timeout = 30

config/mycnf/master_mysql.cnf

@@ -0,0 +1 @@
+# reserved for future use

config/mycnf/master_mysql57.cnf

@@ -0,0 +1,29 @@
+# Options for enabling GTID
+# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html
+gtid_mode = ON
+log_bin
+log_slave_updates
+enforce_gtid_consistency
+
+# Crash-safe replication settings.
+master_info_repository = TABLE
+relay_log_info_repository = TABLE
+relay_log_purge = 1
+relay_log_recovery = 1
+
+# Semi-sync replication is required for automated unplanned failover
+# (when the master goes away). Here we just load the plugin so it's
+# available if desired, but it's disabled at startup.
+#
+# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync
+# at the proper time when replication is set up, or when masters are
+# promoted or demoted.
+plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so
+
+# When semi-sync is enabled, don't allow fallback to async
+# if you get no ack, or have no slaves. This is necessary to
+# prevent alternate futures when doing a failover in response to
+# a master that becomes unresponsive.
+rpl_semi_sync_master_timeout = 1000000000000000000
+rpl_semi_sync_master_wait_no_slave = 1
+

dev.env

@@ -95,16 +95,6 @@ if [[ "$VT_MYSQL_ROOT" == "" ]]; then
   fi
 fi
 
-# restore MYSQL_FLAVOR, saved by bootstrap.sh
-if [ -r "$VTROOT/dist/MYSQL_FLAVOR" ]; then
-  MYSQL_FLAVOR=$(cat "$VTROOT/dist/MYSQL_FLAVOR")
-  export MYSQL_FLAVOR
-fi
-
-# mysql cgo library config
-if [ -z "$MYSQL_FLAVOR" ]; then
-  export MYSQL_FLAVOR=MariaDB
-fi
 PKG_CONFIG_PATH=$(prepend_path "$PKG_CONFIG_PATH" "$VTROOT/lib")
 export PKG_CONFIG_PATH
 

examples/local/env.sh

@@ -29,15 +29,8 @@ if [ -z "$VT_MYSQL_ROOT" ]; then
   export VT_MYSQL_ROOT=$(dirname `dirname $mysql_path`)
 fi
 
-# restore MYSQL_FLAVOR, saved by bootstrap.sh
-if [ -r "$VTROOT/dist/MYSQL_FLAVOR" ]; then
-  MYSQL_FLAVOR=$(cat "$VTROOT/dist/MYSQL_FLAVOR")
-  export MYSQL_FLAVOR
-fi
-
-if [ -z "$MYSQL_FLAVOR" ]; then
-  export MYSQL_FLAVOR=MySQL56
-fi
+# Previously the file specified MYSQL_FLAVOR
+# it is now autodetected
 
 if [ "${TOPO}" = "etcd2" ]; then
     echo "enter etcd2 env"

examples/local/vttablet-up.sh

@@ -38,23 +38,8 @@ source $script_root/env.sh
 
 init_db_sql_file="$VTROOT/config/init_db.sql"
 
-export EXTRA_MY_CNF=$VTROOT/config/mycnf/default-fast.cnf:$VTROOT/config/mycnf/rbr.cnf
-
-case "$MYSQL_FLAVOR" in
-  "MySQL56")
-    export EXTRA_MY_CNF=$EXTRA_MY_CNF:$VTROOT/config/mycnf/master_mysql56.cnf
-    ;;
-  "MariaDB")
-    export EXTRA_MY_CNF=$EXTRA_MY_CNF:$VTROOT/config/mycnf/master_mariadb.cnf
-    ;;
-  "MariaDB103")
-    export EXTRA_MY_CNF=$EXTRA_MY_CNF:$VTROOT/config/mycnf/master_mariadb103.cnf
-    ;;
-  *)
-    echo "Please set MYSQL_FLAVOR to MySQL56 or MariaDB."
-    exit 1
-    ;;
-esac
+# Previously this file set EXTRA_MY_CNF based on MYSQL_FLAVOR
+# and enabled RBR. It now relies on mysqlctl to autodetect
 
 mkdir -p $VTDATAROOT/backups
 

go/cmd/vtcombo/main.go

@@ -99,7 +99,7 @@ func main() {
 	if err != nil {
 		log.Warning(err)
 	}
-	mysqld := mysqlctl.NewMysqld(dbcfgs)
+	mysqld, _ := mysqlctl.NewMysqld(dbcfgs)
 	servenv.OnClose(mysqld.Close)
 
 	// tablets configuration and init.

go/cmd/vttablet/vttablet.go

@@ -117,7 +117,7 @@ func main() {
 	// Create mysqld and register the health reporter (needs to be done
 	// before initializing the agent, so the initial health check
 	// done by the agent has the right reporter)
-	mysqld := mysqlctl.NewMysqld(dbcfgs)
+	mysqld, _ := mysqlctl.NewMysqld(dbcfgs)
 	servenv.OnClose(mysqld.Close)
 
 	// Depends on both query and updateStream.

go/vt/mysqlctl/capabilities.go

@@ -0,0 +1,248 @@
+/*
+Copyright 2019 The Vitess Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ Detect server flavors and capabilities
+*/
+
+package mysqlctl
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+
+	vtenv "vitess.io/vitess/go/vt/env"
+	"vitess.io/vitess/go/vt/log"
+)
+
+type serverVersion []int
+
+const (
+	flavorMySQL   = "mysql"
+	flavorPercona = "percona"
+	flavorMariaDB = "mariadb"
+)
+
+const (
+	CapabilityMySQLUpgradeInServer = 1 << iota
+	CapabilityInitializeInServer
+	CapabilityMySQLxEnabledByDefault
+	CapabilityPersistConfig
+	CapabilityShutdownCommand
+	CapabilityBackupLocks
+	CapabilityDefaultUtf8mb4
+	CapabilitySemiSyncEnabledByDefault
+	CapabilitySystemd // often package specific, not just version specific
+)
+
+var (
+	minimumMySQLUpgradeInServer            = serverVersion{8, 0, 16}
+	minimumMySQLInitializeInServer         = serverVersion{5, 7, 0}
+	minimumMySQLxEnabledByDefault          = serverVersion{8, 0, 11}
+	minimumMySQLPersistConfig              = serverVersion{8, 0, 0}
+	minimumMySQLShutdownCommand            = serverVersion{5, 7, 9}
+	minimumMySQLBackupLocks                = serverVersion{8, 0, 0}
+	minimumMySQLDefaultUtf8mb4             = serverVersion{8, 0, 0}
+	minimumMariaDBSemiSyncEnabledByDefault = serverVersion{10, 3, 3}
+	minimumMariaDBShutdownCommand          = serverVersion{10, 0, 4}
+)
+
+// HasCapability will return a bool value of if the server
+// has a particular feature, based on a known list of
+// introduced versions and detection. This gets more
+// complicated in MySQL 8.0 as several features were introduced
+// after the initial GA release.
+func (mysqld *Mysqld) HasCapability(capability int) bool {
+
+	switch capability {
+	case CapabilityMySQLUpgradeInServer:
+		return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLUpgradeInServer)
+	case CapabilityInitializeInServer:
+		return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLInitializeInServer)
+	case CapabilityMySQLxEnabledByDefault:
+		return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLxEnabledByDefault)
+	case CapabilityPersistConfig:
+		return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLPersistConfig)
+	case CapabilityShutdownCommand:
+		return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLShutdownCommand) || mysqld.IsMariaDB() && mysqld.greaterThan(minimumMariaDBShutdownCommand)
+	case CapabilityBackupLocks:
+		return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLBackupLocks)
+	case CapabilityDefaultUtf8mb4:
+		return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLDefaultUtf8mb4)
+	case CapabilitySemiSyncEnabledByDefault:
+		return mysqld.IsMariaDB() && mysqld.greaterThan(minimumMariaDBSemiSyncEnabledByDefault)
+	case CapabilitySystemd:
+		dir, err := vtenv.VtMysqlRoot()
+		if err != nil {
+			return false
+		}
+		_, err = binaryPath(dir, "mysqld_safe")
+		return err != nil // if there is no mysqld_safe, it means this platform is systemd
+	default:
+		log.Warningf("%d: unknown capability", capability)
+		return false
+	}
+
+}
+
+func (mysqld *Mysqld) greaterThan(version serverVersion) bool {
+	if mysqld.version[0] > version[0] {
+		return true
+	} else if mysqld.version[0] == version[0] && mysqld.version[1] > version[1] {
+		return true
+	} else if mysqld.version[0] == version[0] && mysqld.version[1] == version[1] && mysqld.version[2] >= version[2] {
+		return true
+	} else {
+		return false
+	}
+}
+
+// IsMySQLLike tests if the server is either MySQL
+// or Percona Server. At least currently, Vitess doesn't
+// make use of any specific Percona Server features.
+func (mysqld *Mysqld) IsMySQLLike() bool {
+	return mysqld.flavor == flavorMySQL || mysqld.flavor == flavorPercona
+}
+
+// IsMariaDB tests if the server is MariaDB.
+// IsMySQLLike() and IsMariaDB() are mutually exclusive
+func (mysqld *Mysqld) IsMariaDB() bool {
+	return mysqld.flavor == flavorMariaDB
+}
+
+// MajorVersion returns an integer for the leading version.
+// For example 8 in the case of MySQL 8.0.11, or
+// 5 in the case of 5.6.15
+func (mysqld *Mysqld) MajorVersion() int {
+	return mysqld.version[0]
+}
+
+// MinorVersion returns an integer for the second version
+// For example 6 in the case of 5.6.11
+func (mysqld *Mysqld) MinorVersion() int {
+	return mysqld.version[1]
+}
+
+// PatchVersion returns an integer for the third
+// part of the version. For example 11 in the case of
+// 5.6.11
+func (mysqld *Mysqld) PatchVersion() int {
+	return mysqld.version[2]
+}
+
+func (mysqld *Mysqld) getVersionString() (string, error) {
+
+	mysqlRoot, err := vtenv.VtMysqlRoot()
+	if err != nil {
+		return "", err
+	}
+	mysqldPath, err := binaryPath(mysqlRoot, "mysqld")
+	if err != nil {
+		return "", err
+	}
+
+	_, version, err := execCmd(mysqldPath, []string{"--version"}, nil, mysqlRoot, nil)
+	if err != nil {
+		return "", err
+	}
+
+	return version, nil
+
+}
+
+func (mysqld *Mysqld) detectFlavor() (string, error) {
+
+	mysqlFlavor := os.Getenv("MYSQL_FLAVOR")
+	if len(mysqlFlavor) > 0 {
+		mysqlFlavor = strings.ToLower(mysqlFlavor)[0:5]
+
+		if mysqlFlavor == "mysql" {
+			return flavorMySQL, nil
+		} else if mysqlFlavor == "maria" {
+			return flavorMariaDB, nil
+		}
+	}
+
+	// Flavor was either not defined in ENV, or unknown
+
+	version, err := mysqld.getVersionString()
+	if err != nil {
+		return version, err
+	}
+
+	// Check if the flavor has been defined
+	// This takes first precedence.
+
+	if strings.Contains(version, "MySQL Community Server") {
+		return flavorMySQL, nil
+	} else if strings.Contains(version, "Percona") {
+		return flavorPercona, nil
+	} else if strings.Contains(version, "MariaDB") {
+		return flavorMariaDB, nil
+	}
+
+	return "", fmt.Errorf("Could not autodetect flavor!")
+
+}
+
+func (mysqld *Mysqld) detectVersion() (serverVersion, error) {
+
+	mysqlFlavor := strings.ToLower(os.Getenv("MYSQL_FLAVOR"))
+
+	// Try not to be too smart with version parsing.
+	// Assume GA versions
+	switch mysqlFlavor {
+	case "mysql56":
+		return serverVersion{5, 6, 10}, nil
+	case "mysql57":
+		return serverVersion{5, 7, 9}, nil
+	case "mysql80":
+		return serverVersion{8, 0, 11}, nil
+	case "mariadb10":
+		return serverVersion{10, 0, 10}, nil
+	case "mariadb101":
+		return serverVersion{10, 1, 8}, nil
+	case "mariadb102":
+		return serverVersion{10, 2, 6}, nil
+	case "mariadb103":
+		return serverVersion{10, 3, 7}, nil
+	case "mariadb104":
+		return serverVersion{10, 4, 6}, nil
+	}
+
+	// Could not understand version from ENV
+	// Time to auto-detect instead
+
+	version, err := mysqld.getVersionString()
+	if err != nil {
+		return nil, err
+	}
+
+	// Detect server version
+	re := regexp.MustCompile(`Ver [0-9]+\.[0-9]+\.[0-9]+`)
+	ver := re.Find([]byte(version))[4:]
+	re = regexp.MustCompile(`[0-9]+`)
+	v := re.FindAll([]byte(ver), -1)
+	major, _ := strconv.Atoi(string(v[0]))
+	minor, _ := strconv.Atoi(string(v[1]))
+	patch, _ := strconv.Atoi(string(v[2]))
+
+	return serverVersion{major, minor, patch}, nil
+
+}

go/vt/mysqlctl/cmd.go

@@ -51,7 +51,8 @@ func CreateMysqldAndMycnf(tabletUID uint32, mysqlSocket string, mysqlPort int32)
 		return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err)
 	}
 
-	return NewMysqld(dbcfgs), mycnf, nil
+	mysqld, _ := NewMysqld(dbcfgs)
+	return mysqld, mycnf, nil
 }
 
 // OpenMysqldAndMycnf returns a Mysqld and a Mycnf object to use for working with a MySQL
@@ -69,5 +70,6 @@ func OpenMysqldAndMycnf(tabletUID uint32) (*Mysqld, *Mycnf, error) {
 		return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err)
 	}
 
-	return NewMysqld(dbcfgs), mycnf, nil
+	mysqld, _ := NewMysqld(dbcfgs)
+	return mysqld, mycnf, nil
 }