From f336afe84cd5102d3c36f493100ff2ecbde2513e Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 10 Jul 2019 14:19:12 -0600 Subject: [PATCH 01/18] Add initial support for capabilities This eliminates the need to set MYSQL_FLAVOR Signed-off-by: Morgan Tocker --- config/mycnf/default.cnf | 2 +- config/mycnf/master_mysql.cnf | 1 + config/mycnf/master_mysql57.cnf | 29 ++ dev.env | 10 - examples/local/env.sh | 11 +- examples/local/vttablet-up.sh | 19 +- go/cmd/vtcombo/main.go | 2 +- go/cmd/vttablet/vttablet.go | 2 +- go/vt/mysqlctl/capabilities.go | 248 ++++++++++++++++++ go/vt/mysqlctl/cmd.go | 6 +- go/vt/mysqlctl/mysqld.go | 106 ++++---- .../tabletserver/vstreamer/testenv/testenv.go | 2 +- 12 files changed, 346 insertions(+), 92 deletions(-) create mode 100644 config/mycnf/master_mysql.cnf create mode 100644 config/mycnf/master_mysql57.cnf create mode 100644 go/vt/mysqlctl/capabilities.go diff --git a/config/mycnf/default.cnf b/config/mycnf/default.cnf index cb6711d3fa7..2fe69d72def 100644 --- a/config/mycnf/default.cnf +++ b/config/mycnf/default.cnf @@ -2,7 +2,7 @@ sql_mode = STRICT_TRANS_TABLES back_log = 50 -binlog_format = statement +binlog_format = row character_set_server = utf8 collation_server = utf8_general_ci connect_timeout = 30 diff --git a/config/mycnf/master_mysql.cnf b/config/mycnf/master_mysql.cnf new file mode 100644 index 00000000000..43c55130bc1 --- /dev/null +++ b/config/mycnf/master_mysql.cnf @@ -0,0 +1 @@ +# reserved for future use diff --git a/config/mycnf/master_mysql57.cnf b/config/mycnf/master_mysql57.cnf new file mode 100644 index 00000000000..7c62eabc2d9 --- /dev/null +++ b/config/mycnf/master_mysql57.cnf @@ -0,0 +1,29 @@ +# Options for enabling GTID +# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html +gtid_mode = ON +log_bin +log_slave_updates +enforce_gtid_consistency + +# Crash-safe replication settings. +master_info_repository = TABLE +relay_log_info_repository = TABLE +relay_log_purge = 1 +relay_log_recovery = 1 + +# Semi-sync replication is required for automated unplanned failover +# (when the master goes away). Here we just load the plugin so it's +# available if desired, but it's disabled at startup. +# +# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync +# at the proper time when replication is set up, or when masters are +# promoted or demoted. +plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so + +# When semi-sync is enabled, don't allow fallback to async +# if you get no ack, or have no slaves. This is necessary to +# prevent alternate futures when doing a failover in response to +# a master that becomes unresponsive. +rpl_semi_sync_master_timeout = 1000000000000000000 +rpl_semi_sync_master_wait_no_slave = 1 + diff --git a/dev.env b/dev.env index e31c16167e2..ca7bc608721 100644 --- a/dev.env +++ b/dev.env @@ -95,16 +95,6 @@ if [[ "$VT_MYSQL_ROOT" == "" ]]; then fi fi -# restore MYSQL_FLAVOR, saved by bootstrap.sh -if [ -r "$VTROOT/dist/MYSQL_FLAVOR" ]; then - MYSQL_FLAVOR=$(cat "$VTROOT/dist/MYSQL_FLAVOR") - export MYSQL_FLAVOR -fi - -# mysql cgo library config -if [ -z "$MYSQL_FLAVOR" ]; then - export MYSQL_FLAVOR=MariaDB -fi PKG_CONFIG_PATH=$(prepend_path "$PKG_CONFIG_PATH" "$VTROOT/lib") export PKG_CONFIG_PATH diff --git a/examples/local/env.sh b/examples/local/env.sh index 608725a2487..f285de810ee 100644 --- a/examples/local/env.sh +++ b/examples/local/env.sh @@ -29,15 +29,8 @@ if [ -z "$VT_MYSQL_ROOT" ]; then export VT_MYSQL_ROOT=$(dirname `dirname $mysql_path`) fi -# restore MYSQL_FLAVOR, saved by bootstrap.sh -if [ -r "$VTROOT/dist/MYSQL_FLAVOR" ]; then - MYSQL_FLAVOR=$(cat "$VTROOT/dist/MYSQL_FLAVOR") - export MYSQL_FLAVOR -fi - -if [ -z "$MYSQL_FLAVOR" ]; then - export MYSQL_FLAVOR=MySQL56 -fi +# Previously the file specified MYSQL_FLAVOR +# it is now autodetected if [ "${TOPO}" = "etcd2" ]; then echo "enter etcd2 env" diff --git a/examples/local/vttablet-up.sh b/examples/local/vttablet-up.sh index 0acd5e51b9f..013b657942f 100755 --- a/examples/local/vttablet-up.sh +++ b/examples/local/vttablet-up.sh @@ -38,23 +38,8 @@ source $script_root/env.sh init_db_sql_file="$VTROOT/config/init_db.sql" -export EXTRA_MY_CNF=$VTROOT/config/mycnf/default-fast.cnf:$VTROOT/config/mycnf/rbr.cnf - -case "$MYSQL_FLAVOR" in - "MySQL56") - export EXTRA_MY_CNF=$EXTRA_MY_CNF:$VTROOT/config/mycnf/master_mysql56.cnf - ;; - "MariaDB") - export EXTRA_MY_CNF=$EXTRA_MY_CNF:$VTROOT/config/mycnf/master_mariadb.cnf - ;; - "MariaDB103") - export EXTRA_MY_CNF=$EXTRA_MY_CNF:$VTROOT/config/mycnf/master_mariadb103.cnf - ;; - *) - echo "Please set MYSQL_FLAVOR to MySQL56 or MariaDB." - exit 1 - ;; -esac +# Previously this file set EXTRA_MY_CNF based on MYSQL_FLAVOR +# and enabled RBR. It now relies on mysqlctl to autodetect mkdir -p $VTDATAROOT/backups diff --git a/go/cmd/vtcombo/main.go b/go/cmd/vtcombo/main.go index f92181861f4..2bd278b09f4 100644 --- a/go/cmd/vtcombo/main.go +++ b/go/cmd/vtcombo/main.go @@ -99,7 +99,7 @@ func main() { if err != nil { log.Warning(err) } - mysqld := mysqlctl.NewMysqld(dbcfgs) + mysqld, _ := mysqlctl.NewMysqld(dbcfgs) servenv.OnClose(mysqld.Close) // tablets configuration and init. diff --git a/go/cmd/vttablet/vttablet.go b/go/cmd/vttablet/vttablet.go index 76493fc881a..09ae45ff507 100644 --- a/go/cmd/vttablet/vttablet.go +++ b/go/cmd/vttablet/vttablet.go @@ -117,7 +117,7 @@ func main() { // Create mysqld and register the health reporter (needs to be done // before initializing the agent, so the initial health check // done by the agent has the right reporter) - mysqld := mysqlctl.NewMysqld(dbcfgs) + mysqld, _ := mysqlctl.NewMysqld(dbcfgs) servenv.OnClose(mysqld.Close) // Depends on both query and updateStream. diff --git a/go/vt/mysqlctl/capabilities.go b/go/vt/mysqlctl/capabilities.go new file mode 100644 index 00000000000..34cd4d73591 --- /dev/null +++ b/go/vt/mysqlctl/capabilities.go @@ -0,0 +1,248 @@ +/* +Copyright 2019 The Vitess Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + Detect server flavors and capabilities +*/ + +package mysqlctl + +import ( + "fmt" + "os" + "regexp" + "strconv" + "strings" + + vtenv "vitess.io/vitess/go/vt/env" + "vitess.io/vitess/go/vt/log" +) + +type serverVersion []int + +const ( + flavorMySQL = "mysql" + flavorPercona = "percona" + flavorMariaDB = "mariadb" +) + +const ( + CapabilityMySQLUpgradeInServer = 1 << iota + CapabilityInitializeInServer + CapabilityMySQLxEnabledByDefault + CapabilityPersistConfig + CapabilityShutdownCommand + CapabilityBackupLocks + CapabilityDefaultUtf8mb4 + CapabilitySemiSyncEnabledByDefault + CapabilitySystemd // often package specific, not just version specific +) + +var ( + minimumMySQLUpgradeInServer = serverVersion{8, 0, 16} + minimumMySQLInitializeInServer = serverVersion{5, 7, 0} + minimumMySQLxEnabledByDefault = serverVersion{8, 0, 11} + minimumMySQLPersistConfig = serverVersion{8, 0, 0} + minimumMySQLShutdownCommand = serverVersion{5, 7, 9} + minimumMySQLBackupLocks = serverVersion{8, 0, 0} + minimumMySQLDefaultUtf8mb4 = serverVersion{8, 0, 0} + minimumMariaDBSemiSyncEnabledByDefault = serverVersion{10, 3, 3} + minimumMariaDBShutdownCommand = serverVersion{10, 0, 4} +) + +// HasCapability will return a bool value of if the server +// has a particular feature, based on a known list of +// introduced versions and detection. This gets more +// complicated in MySQL 8.0 as several features were introduced +// after the initial GA release. +func (mysqld *Mysqld) HasCapability(capability int) bool { + + switch capability { + case CapabilityMySQLUpgradeInServer: + return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLUpgradeInServer) + case CapabilityInitializeInServer: + return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLInitializeInServer) + case CapabilityMySQLxEnabledByDefault: + return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLxEnabledByDefault) + case CapabilityPersistConfig: + return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLPersistConfig) + case CapabilityShutdownCommand: + return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLShutdownCommand) || mysqld.IsMariaDB() && mysqld.greaterThan(minimumMariaDBShutdownCommand) + case CapabilityBackupLocks: + return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLBackupLocks) + case CapabilityDefaultUtf8mb4: + return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLDefaultUtf8mb4) + case CapabilitySemiSyncEnabledByDefault: + return mysqld.IsMariaDB() && mysqld.greaterThan(minimumMariaDBSemiSyncEnabledByDefault) + case CapabilitySystemd: + dir, err := vtenv.VtMysqlRoot() + if err != nil { + return false + } + _, err = binaryPath(dir, "mysqld_safe") + return err != nil // if there is no mysqld_safe, it means this platform is systemd + default: + log.Warningf("%d: unknown capability", capability) + return false + } + +} + +func (mysqld *Mysqld) greaterThan(version serverVersion) bool { + if mysqld.version[0] > version[0] { + return true + } else if mysqld.version[0] == version[0] && mysqld.version[1] > version[1] { + return true + } else if mysqld.version[0] == version[0] && mysqld.version[1] == version[1] && mysqld.version[2] >= version[2] { + return true + } else { + return false + } +} + +// IsMySQLLike tests if the server is either MySQL +// or Percona Server. At least currently, Vitess doesn't +// make use of any specific Percona Server features. +func (mysqld *Mysqld) IsMySQLLike() bool { + return mysqld.flavor == flavorMySQL || mysqld.flavor == flavorPercona +} + +// IsMariaDB tests if the server is MariaDB. +// IsMySQLLike() and IsMariaDB() are mutually exclusive +func (mysqld *Mysqld) IsMariaDB() bool { + return mysqld.flavor == flavorMariaDB +} + +// MajorVersion returns an integer for the leading version. +// For example 8 in the case of MySQL 8.0.11, or +// 5 in the case of 5.6.15 +func (mysqld *Mysqld) MajorVersion() int { + return mysqld.version[0] +} + +// MinorVersion returns an integer for the second version +// For example 6 in the case of 5.6.11 +func (mysqld *Mysqld) MinorVersion() int { + return mysqld.version[1] +} + +// PatchVersion returns an integer for the third +// part of the version. For example 11 in the case of +// 5.6.11 +func (mysqld *Mysqld) PatchVersion() int { + return mysqld.version[2] +} + +func (mysqld *Mysqld) getVersionString() (string, error) { + + mysqlRoot, err := vtenv.VtMysqlRoot() + if err != nil { + return "", err + } + mysqldPath, err := binaryPath(mysqlRoot, "mysqld") + if err != nil { + return "", err + } + + _, version, err := execCmd(mysqldPath, []string{"--version"}, nil, mysqlRoot, nil) + if err != nil { + return "", err + } + + return version, nil + +} + +func (mysqld *Mysqld) detectFlavor() (string, error) { + + mysqlFlavor := os.Getenv("MYSQL_FLAVOR") + if len(mysqlFlavor) > 0 { + mysqlFlavor = strings.ToLower(mysqlFlavor)[0:5] + + if mysqlFlavor == "mysql" { + return flavorMySQL, nil + } else if mysqlFlavor == "maria" { + return flavorMariaDB, nil + } + } + + // Flavor was either not defined in ENV, or unknown + + version, err := mysqld.getVersionString() + if err != nil { + return version, err + } + + // Check if the flavor has been defined + // This takes first precedence. + + if strings.Contains(version, "MySQL Community Server") { + return flavorMySQL, nil + } else if strings.Contains(version, "Percona") { + return flavorPercona, nil + } else if strings.Contains(version, "MariaDB") { + return flavorMariaDB, nil + } + + return "", fmt.Errorf("Could not autodetect flavor!") + +} + +func (mysqld *Mysqld) detectVersion() (serverVersion, error) { + + mysqlFlavor := strings.ToLower(os.Getenv("MYSQL_FLAVOR")) + + // Try not to be too smart with version parsing. + // Assume GA versions + switch mysqlFlavor { + case "mysql56": + return serverVersion{5, 6, 10}, nil + case "mysql57": + return serverVersion{5, 7, 9}, nil + case "mysql80": + return serverVersion{8, 0, 11}, nil + case "mariadb10": + return serverVersion{10, 0, 10}, nil + case "mariadb101": + return serverVersion{10, 1, 8}, nil + case "mariadb102": + return serverVersion{10, 2, 6}, nil + case "mariadb103": + return serverVersion{10, 3, 7}, nil + case "mariadb104": + return serverVersion{10, 4, 6}, nil + } + + // Could not understand version from ENV + // Time to auto-detect instead + + version, err := mysqld.getVersionString() + if err != nil { + return nil, err + } + + // Detect server version + re := regexp.MustCompile(`Ver [0-9]+\.[0-9]+\.[0-9]+`) + ver := re.Find([]byte(version))[4:] + re = regexp.MustCompile(`[0-9]+`) + v := re.FindAll([]byte(ver), -1) + major, _ := strconv.Atoi(string(v[0])) + minor, _ := strconv.Atoi(string(v[1])) + patch, _ := strconv.Atoi(string(v[2])) + + return serverVersion{major, minor, patch}, nil + +} diff --git a/go/vt/mysqlctl/cmd.go b/go/vt/mysqlctl/cmd.go index 7289d999570..95701b388d4 100644 --- a/go/vt/mysqlctl/cmd.go +++ b/go/vt/mysqlctl/cmd.go @@ -51,7 +51,8 @@ func CreateMysqldAndMycnf(tabletUID uint32, mysqlSocket string, mysqlPort int32) return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err) } - return NewMysqld(dbcfgs), mycnf, nil + mysqld, _ := NewMysqld(dbcfgs) + return mysqld, mycnf, nil } // OpenMysqldAndMycnf returns a Mysqld and a Mycnf object to use for working with a MySQL @@ -69,5 +70,6 @@ func OpenMysqldAndMycnf(tabletUID uint32) (*Mysqld, *Mycnf, error) { return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err) } - return NewMysqld(dbcfgs), mycnf, nil + mysqld, _ := NewMysqld(dbcfgs) + return mysqld, mycnf, nil } diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 029586d1c15..4501b184477 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -25,6 +25,7 @@ package mysqlctl import ( "bufio" + "bytes" "errors" "flag" "fmt" @@ -38,8 +39,6 @@ import ( "sync" "time" - "bytes" - "golang.org/x/net/context" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/stats" @@ -86,6 +85,9 @@ type Mysqld struct { dbaPool *dbconnpool.ConnectionPool appPool *dbconnpool.ConnectionPool + flavor string + version serverVersion + // mutex protects the fields below. mutex sync.Mutex onTermFuncs []func() @@ -94,7 +96,7 @@ type Mysqld struct { // NewMysqld creates a Mysqld object based on the provided configuration // and connection parameters. -func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { +func NewMysqld(dbcfgs *dbconfigs.DBConfigs) (*Mysqld, error) { result := &Mysqld{ dbcfgs: dbcfgs, } @@ -107,7 +109,19 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { result.appPool = dbconnpool.NewConnectionPool("AppConnPool", *appPoolSize, *appIdleTimeout, *poolDynamicHostnameResolution) result.appPool.Open(dbcfgs.AppWithDB(), appMysqlStats) - return result + var err error + result.flavor, err = result.detectFlavor() + if err != nil { + return result, err + } + + result.version, err = result.detectVersion() + if err != nil { + return result, err + } + + log.Infof("mysqld is flavor: %s, version: %v", result.flavor, result.version) + return result, nil } // RunMysqlUpgrade will run the mysql_upgrade program on the current @@ -125,6 +139,11 @@ func (mysqld *Mysqld) RunMysqlUpgrade() error { return client.RunMysqlUpgrade(context.TODO()) } + if mysqld.HasCapability(CapabilityMySQLUpgradeInServer) { + log.Warningf("MySQL version has built-in upgrade, skipping RunMySQLUpgrade") + return nil + } + // Find mysql_upgrade. If not there, we do nothing. dir, err := vtenv.VtMysqlRoot() if err != nil { @@ -209,19 +228,31 @@ func (mysqld *Mysqld) startNoWait(ctx context.Context, cnf *Mycnf, mysqldArgs .. if err != nil { return err } - name, err = binaryPath(dir, "mysqld_safe") - if err != nil { - // The movement to use systemd means that mysqld_safe is not always provided. - // This should not be considered an issue do not generate a warning. - log.Infof("%v: trying to launch mysqld instead", err) + + if mysqld.HasCapability(CapabilitySystemd) { + // The capabilities system did not detect mysqld_safe. + // It is assumed that this mysqld package is systemd enabled. + log.Info("launching mysqld instead of mysqld_safe") name, err = binaryPath(dir, "mysqld") - // If this also fails, return an error. if err != nil { return err } + } else { + name, err = binaryPath(dir, "mysqld_safe") + if err != nil { + return err + } + } + + mysqlBaseDir, err := vtenv.VtMysqlBaseDir() + if err != nil { + return err } + arg := []string{ - "--defaults-file=" + cnf.path} + "--defaults-file=" + cnf.path, + "--basedir=" + mysqlBaseDir, + } arg = append(arg, mysqldArgs...) env := []string{os.ExpandEnv("LD_LIBRARY_PATH=$VT_MYSQL_ROOT/lib/mysql")} @@ -534,13 +565,6 @@ func (mysqld *Mysqld) Init(ctx context.Context, cnf *Mycnf, initDBSQLFile string return nil } -// MySQL 5.7 GA and up have deprecated mysql_install_db. -// Instead, initialization is built into mysqld. -func useMysqldInitialize(version string) bool { - return strings.Contains(version, "Ver 5.7.") || - strings.Contains(version, "Ver 8.0.") -} - func (mysqld *Mysqld) installDataDir(cnf *Mycnf) error { mysqlRoot, err := vtenv.VtMysqlRoot() if err != nil { @@ -556,15 +580,8 @@ func (mysqld *Mysqld) installDataDir(cnf *Mycnf) error { return err } - // Check mysqld version. - _, version, err := execCmd(mysqldPath, []string{"--version"}, nil, mysqlRoot, nil) - if err != nil { - return err - } - - if useMysqldInitialize(version) { + if mysqld.HasCapability(CapabilityInitializeInServer) { log.Infof("Installing data dir with mysqld --initialize-insecure") - args := []string{ "--defaults-file=" + cnf.path, "--basedir=" + mysqlBaseDir, @@ -606,7 +623,7 @@ func (mysqld *Mysqld) initConfig(root string, cnf *Mycnf, outFile string) error switch hr := hook.NewHookWithEnv("make_mycnf", nil, env).Execute(); hr.ExitStatus { case hook.HOOK_DOES_NOT_EXIST: log.Infof("make_mycnf hook doesn't exist, reading template files") - configData, err = cnf.makeMycnf(getMycnfTemplates(root)) + configData, err = cnf.makeMycnf(mysqld.getMycnfTemplates(root)) case hook.HOOK_SUCCESS: configData, err = cnf.fillMycnfTemplate(hr.Stdout) default: @@ -628,7 +645,8 @@ func contains(haystack []string, needle string) bool { return false } -func getMycnfTemplates(root string) []string { +func (mysqld *Mysqld) getMycnfTemplates(root string) []string { + if *mycnfTemplateFile != "" { return []string{*mycnfTemplateFile} } @@ -644,28 +662,16 @@ func getMycnfTemplates(root string) []string { cnfTemplatePaths = append(cnfTemplatePaths, parts...) } - switch mysqlFlavor := os.Getenv("MYSQL_FLAVOR"); mysqlFlavor { - case "MariaDB": - path := path.Join(root, "config/mycnf/master_mariadb.cnf") - if !contains(cnfTemplatePaths, path) { - cnfTemplatePaths = append(cnfTemplatePaths, path) - } - case "MariaDB103": - path := path.Join(root, "config/mycnf/master_mariadb103.cnf") - if !contains(cnfTemplatePaths, path) { - cnfTemplatePaths = append(cnfTemplatePaths, path) - } - case "MySQL80": - path := path.Join(root, "config/mycnf/master_mysql80.cnf") - if !contains(cnfTemplatePaths, path) { - cnfTemplatePaths = append(cnfTemplatePaths, path) - } - default: - path := path.Join(root, "config/mycnf/master_mysql56.cnf") - // By default we assume Mysql56 compatable - if !contains(cnfTemplatePaths, path) { - cnfTemplatePaths = append(cnfTemplatePaths, path) - } + // master_{flavor}.cnf + p := path.Join(root, fmt.Sprintf("config/mycnf/master_%s.cnf", mysqld.flavor)) + if !contains(cnfTemplatePaths, p) { + cnfTemplatePaths = append(cnfTemplatePaths, p) + } + + // master_{flavor}{major}{minor}.cnf + p = path.Join(root, fmt.Sprintf("config/mycnf/master_%s%d%d.cnf", mysqld.flavor, mysqld.MajorVersion(), mysqld.MinorVersion())) + if !contains(cnfTemplatePaths, p) { + cnfTemplatePaths = append(cnfTemplatePaths, p) } return cnfTemplatePaths diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index be2eb3070fe..a39abb28489 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -104,7 +104,7 @@ func Init() (*Env, error) { } te.Dbcfgs = dbconfigs.NewTestDBConfigs(te.cluster.MySQLConnParams(), te.cluster.MySQLAppDebugConnParams(), te.cluster.DbName()) - te.Mysqld = mysqlctl.NewMysqld(te.Dbcfgs) + te.Mysqld, _ = mysqlctl.NewMysqld(te.Dbcfgs) te.SchemaEngine = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) te.SchemaEngine.InitDBConfig(te.Dbcfgs) From 030c2fd447f4a0101916411dd253c45ad35c59f5 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Thu, 11 Jul 2019 10:46:59 -0600 Subject: [PATCH 02/18] Removed MYSQL_FLAVOR variable precedence It will cause backwards compatibility problems, since many users set "MySQL56" for any MySQL system. Signed-off-by: Morgan Tocker --- config/mycnf/master_mariadb.cnf | 10 -------- config/mycnf/master_mariadb100.cnf | 10 ++++++++ config/mycnf/master_mariadb101.cnf | 10 ++++++++ config/mycnf/master_mariadb102.cnf | 10 ++++++++ config/mycnf/master_mysql.cnf | 1 - go/vt/mysqlctl/capabilities.go | 40 ------------------------------ go/vt/mysqlctl/mysqld.go | 20 +++++++++++---- 7 files changed, 45 insertions(+), 56 deletions(-) create mode 100644 config/mycnf/master_mariadb100.cnf create mode 100644 config/mycnf/master_mariadb101.cnf create mode 100644 config/mycnf/master_mariadb102.cnf delete mode 100644 config/mycnf/master_mysql.cnf diff --git a/config/mycnf/master_mariadb.cnf b/config/mycnf/master_mariadb.cnf index 83f6f318e4c..064e1307964 100644 --- a/config/mycnf/master_mariadb.cnf +++ b/config/mycnf/master_mariadb.cnf @@ -1,16 +1,6 @@ # enable strict mode so it's safe to compare sequence numbers across different server IDs. gtid_strict_mode = 1 innodb_stats_persistent = 0 -innodb_support_xa = 0 - -# Semi-sync replication is required for automated unplanned failover -# (when the master goes away). Here we just load the plugin so it's -# available if desired, but it's disabled at startup. -# -# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync -# at the proper time when replication is set up, or when masters are -# promoted or demoted. -plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so # When semi-sync is enabled, don't allow fallback to async # if you get no ack, or have no slaves. This is necessary to diff --git a/config/mycnf/master_mariadb100.cnf b/config/mycnf/master_mariadb100.cnf new file mode 100644 index 00000000000..9f61d14d4be --- /dev/null +++ b/config/mycnf/master_mariadb100.cnf @@ -0,0 +1,10 @@ +innodb_support_xa = 0 + +# Semi-sync replication is required for automated unplanned failover +# (when the master goes away). Here we just load the plugin so it's +# available if desired, but it's disabled at startup. +# +# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync +# at the proper time when replication is set up, or when masters are +# promoted or demoted. +plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so diff --git a/config/mycnf/master_mariadb101.cnf b/config/mycnf/master_mariadb101.cnf new file mode 100644 index 00000000000..9f61d14d4be --- /dev/null +++ b/config/mycnf/master_mariadb101.cnf @@ -0,0 +1,10 @@ +innodb_support_xa = 0 + +# Semi-sync replication is required for automated unplanned failover +# (when the master goes away). Here we just load the plugin so it's +# available if desired, but it's disabled at startup. +# +# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync +# at the proper time when replication is set up, or when masters are +# promoted or demoted. +plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so diff --git a/config/mycnf/master_mariadb102.cnf b/config/mycnf/master_mariadb102.cnf new file mode 100644 index 00000000000..9f61d14d4be --- /dev/null +++ b/config/mycnf/master_mariadb102.cnf @@ -0,0 +1,10 @@ +innodb_support_xa = 0 + +# Semi-sync replication is required for automated unplanned failover +# (when the master goes away). Here we just load the plugin so it's +# available if desired, but it's disabled at startup. +# +# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync +# at the proper time when replication is set up, or when masters are +# promoted or demoted. +plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so diff --git a/config/mycnf/master_mysql.cnf b/config/mycnf/master_mysql.cnf deleted file mode 100644 index 43c55130bc1..00000000000 --- a/config/mycnf/master_mysql.cnf +++ /dev/null @@ -1 +0,0 @@ -# reserved for future use diff --git a/go/vt/mysqlctl/capabilities.go b/go/vt/mysqlctl/capabilities.go index 34cd4d73591..552524d5eda 100644 --- a/go/vt/mysqlctl/capabilities.go +++ b/go/vt/mysqlctl/capabilities.go @@ -22,7 +22,6 @@ package mysqlctl import ( "fmt" - "os" "regexp" "strconv" "strings" @@ -168,19 +167,6 @@ func (mysqld *Mysqld) getVersionString() (string, error) { func (mysqld *Mysqld) detectFlavor() (string, error) { - mysqlFlavor := os.Getenv("MYSQL_FLAVOR") - if len(mysqlFlavor) > 0 { - mysqlFlavor = strings.ToLower(mysqlFlavor)[0:5] - - if mysqlFlavor == "mysql" { - return flavorMySQL, nil - } else if mysqlFlavor == "maria" { - return flavorMariaDB, nil - } - } - - // Flavor was either not defined in ENV, or unknown - version, err := mysqld.getVersionString() if err != nil { return version, err @@ -203,32 +189,6 @@ func (mysqld *Mysqld) detectFlavor() (string, error) { func (mysqld *Mysqld) detectVersion() (serverVersion, error) { - mysqlFlavor := strings.ToLower(os.Getenv("MYSQL_FLAVOR")) - - // Try not to be too smart with version parsing. - // Assume GA versions - switch mysqlFlavor { - case "mysql56": - return serverVersion{5, 6, 10}, nil - case "mysql57": - return serverVersion{5, 7, 9}, nil - case "mysql80": - return serverVersion{8, 0, 11}, nil - case "mariadb10": - return serverVersion{10, 0, 10}, nil - case "mariadb101": - return serverVersion{10, 1, 8}, nil - case "mariadb102": - return serverVersion{10, 2, 6}, nil - case "mariadb103": - return serverVersion{10, 3, 7}, nil - case "mariadb104": - return serverVersion{10, 4, 6}, nil - } - - // Could not understand version from ENV - // Time to auto-detect instead - version, err := mysqld.getVersionString() if err != nil { return nil, err diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 4501b184477..3688fcfb53b 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -484,7 +484,7 @@ func execCmd(name string, args, env []string, dir string, input io.Reader) (cmd // binaryPath does a limited path lookup for a command, // searching only within sbin and bin in the given root. func binaryPath(root, binary string) (string, error) { - subdirs := []string{"sbin", "bin", "libexec"} + subdirs := []string{"sbin", "bin", "libexec", "scripts"} for _, subdir := range subdirs { binPath := path.Join(root, subdir, binary) if _, err := os.Stat(binPath); err == nil { @@ -662,15 +662,25 @@ func (mysqld *Mysqld) getMycnfTemplates(root string) []string { cnfTemplatePaths = append(cnfTemplatePaths, parts...) } + // Only include these files if they exist. // master_{flavor}.cnf - p := path.Join(root, fmt.Sprintf("config/mycnf/master_%s.cnf", mysqld.flavor)) - if !contains(cnfTemplatePaths, p) { + // Percona Server == MySQL in this context + + f := flavorMariaDB + if mysqld.IsMySQLLike() { + f = flavorMySQL + } + + p := path.Join(root, fmt.Sprintf("config/mycnf/master_%s.cnf", f)) + _, err := os.Stat(p) + if err == nil && !contains(cnfTemplatePaths, p) { cnfTemplatePaths = append(cnfTemplatePaths, p) } // master_{flavor}{major}{minor}.cnf - p = path.Join(root, fmt.Sprintf("config/mycnf/master_%s%d%d.cnf", mysqld.flavor, mysqld.MajorVersion(), mysqld.MinorVersion())) - if !contains(cnfTemplatePaths, p) { + p = path.Join(root, fmt.Sprintf("config/mycnf/master_%s%d%d.cnf", f, mysqld.MajorVersion(), mysqld.MinorVersion())) + _, err = os.Stat(p) + if err == nil && !contains(cnfTemplatePaths, p) { cnfTemplatePaths = append(cnfTemplatePaths, p) } From 118f3e78f06835ec60f0f31694fec6d8a89ab63d Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Sun, 14 Jul 2019 17:49:46 -0700 Subject: [PATCH 03/18] Address PR feedback (WIP) Signed-off-by: Morgan Tocker --- go/cmd/vtcombo/main.go | 2 +- go/cmd/vttablet/vttablet.go | 2 +- go/vt/mysqlctl/capabilities.go | 208 ------------------ go/vt/mysqlctl/capabilityset.go | 78 +++++++ go/vt/mysqlctl/cmd.go | 4 +- go/vt/mysqlctl/mysqld.go | 38 ++-- go/vt/mysqlctl/version.go | 37 ++++ .../tabletserver/vstreamer/testenv/testenv.go | 2 +- 8 files changed, 139 insertions(+), 232 deletions(-) delete mode 100644 go/vt/mysqlctl/capabilities.go create mode 100644 go/vt/mysqlctl/capabilityset.go create mode 100644 go/vt/mysqlctl/version.go diff --git a/go/cmd/vtcombo/main.go b/go/cmd/vtcombo/main.go index 2bd278b09f4..f92181861f4 100644 --- a/go/cmd/vtcombo/main.go +++ b/go/cmd/vtcombo/main.go @@ -99,7 +99,7 @@ func main() { if err != nil { log.Warning(err) } - mysqld, _ := mysqlctl.NewMysqld(dbcfgs) + mysqld := mysqlctl.NewMysqld(dbcfgs) servenv.OnClose(mysqld.Close) // tablets configuration and init. diff --git a/go/cmd/vttablet/vttablet.go b/go/cmd/vttablet/vttablet.go index 09ae45ff507..76493fc881a 100644 --- a/go/cmd/vttablet/vttablet.go +++ b/go/cmd/vttablet/vttablet.go @@ -117,7 +117,7 @@ func main() { // Create mysqld and register the health reporter (needs to be done // before initializing the agent, so the initial health check // done by the agent has the right reporter) - mysqld, _ := mysqlctl.NewMysqld(dbcfgs) + mysqld := mysqlctl.NewMysqld(dbcfgs) servenv.OnClose(mysqld.Close) // Depends on both query and updateStream. diff --git a/go/vt/mysqlctl/capabilities.go b/go/vt/mysqlctl/capabilities.go deleted file mode 100644 index 552524d5eda..00000000000 --- a/go/vt/mysqlctl/capabilities.go +++ /dev/null @@ -1,208 +0,0 @@ -/* -Copyright 2019 The Vitess Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* - Detect server flavors and capabilities -*/ - -package mysqlctl - -import ( - "fmt" - "regexp" - "strconv" - "strings" - - vtenv "vitess.io/vitess/go/vt/env" - "vitess.io/vitess/go/vt/log" -) - -type serverVersion []int - -const ( - flavorMySQL = "mysql" - flavorPercona = "percona" - flavorMariaDB = "mariadb" -) - -const ( - CapabilityMySQLUpgradeInServer = 1 << iota - CapabilityInitializeInServer - CapabilityMySQLxEnabledByDefault - CapabilityPersistConfig - CapabilityShutdownCommand - CapabilityBackupLocks - CapabilityDefaultUtf8mb4 - CapabilitySemiSyncEnabledByDefault - CapabilitySystemd // often package specific, not just version specific -) - -var ( - minimumMySQLUpgradeInServer = serverVersion{8, 0, 16} - minimumMySQLInitializeInServer = serverVersion{5, 7, 0} - minimumMySQLxEnabledByDefault = serverVersion{8, 0, 11} - minimumMySQLPersistConfig = serverVersion{8, 0, 0} - minimumMySQLShutdownCommand = serverVersion{5, 7, 9} - minimumMySQLBackupLocks = serverVersion{8, 0, 0} - minimumMySQLDefaultUtf8mb4 = serverVersion{8, 0, 0} - minimumMariaDBSemiSyncEnabledByDefault = serverVersion{10, 3, 3} - minimumMariaDBShutdownCommand = serverVersion{10, 0, 4} -) - -// HasCapability will return a bool value of if the server -// has a particular feature, based on a known list of -// introduced versions and detection. This gets more -// complicated in MySQL 8.0 as several features were introduced -// after the initial GA release. -func (mysqld *Mysqld) HasCapability(capability int) bool { - - switch capability { - case CapabilityMySQLUpgradeInServer: - return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLUpgradeInServer) - case CapabilityInitializeInServer: - return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLInitializeInServer) - case CapabilityMySQLxEnabledByDefault: - return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLxEnabledByDefault) - case CapabilityPersistConfig: - return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLPersistConfig) - case CapabilityShutdownCommand: - return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLShutdownCommand) || mysqld.IsMariaDB() && mysqld.greaterThan(minimumMariaDBShutdownCommand) - case CapabilityBackupLocks: - return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLBackupLocks) - case CapabilityDefaultUtf8mb4: - return mysqld.IsMySQLLike() && mysqld.greaterThan(minimumMySQLDefaultUtf8mb4) - case CapabilitySemiSyncEnabledByDefault: - return mysqld.IsMariaDB() && mysqld.greaterThan(minimumMariaDBSemiSyncEnabledByDefault) - case CapabilitySystemd: - dir, err := vtenv.VtMysqlRoot() - if err != nil { - return false - } - _, err = binaryPath(dir, "mysqld_safe") - return err != nil // if there is no mysqld_safe, it means this platform is systemd - default: - log.Warningf("%d: unknown capability", capability) - return false - } - -} - -func (mysqld *Mysqld) greaterThan(version serverVersion) bool { - if mysqld.version[0] > version[0] { - return true - } else if mysqld.version[0] == version[0] && mysqld.version[1] > version[1] { - return true - } else if mysqld.version[0] == version[0] && mysqld.version[1] == version[1] && mysqld.version[2] >= version[2] { - return true - } else { - return false - } -} - -// IsMySQLLike tests if the server is either MySQL -// or Percona Server. At least currently, Vitess doesn't -// make use of any specific Percona Server features. -func (mysqld *Mysqld) IsMySQLLike() bool { - return mysqld.flavor == flavorMySQL || mysqld.flavor == flavorPercona -} - -// IsMariaDB tests if the server is MariaDB. -// IsMySQLLike() and IsMariaDB() are mutually exclusive -func (mysqld *Mysqld) IsMariaDB() bool { - return mysqld.flavor == flavorMariaDB -} - -// MajorVersion returns an integer for the leading version. -// For example 8 in the case of MySQL 8.0.11, or -// 5 in the case of 5.6.15 -func (mysqld *Mysqld) MajorVersion() int { - return mysqld.version[0] -} - -// MinorVersion returns an integer for the second version -// For example 6 in the case of 5.6.11 -func (mysqld *Mysqld) MinorVersion() int { - return mysqld.version[1] -} - -// PatchVersion returns an integer for the third -// part of the version. For example 11 in the case of -// 5.6.11 -func (mysqld *Mysqld) PatchVersion() int { - return mysqld.version[2] -} - -func (mysqld *Mysqld) getVersionString() (string, error) { - - mysqlRoot, err := vtenv.VtMysqlRoot() - if err != nil { - return "", err - } - mysqldPath, err := binaryPath(mysqlRoot, "mysqld") - if err != nil { - return "", err - } - - _, version, err := execCmd(mysqldPath, []string{"--version"}, nil, mysqlRoot, nil) - if err != nil { - return "", err - } - - return version, nil - -} - -func (mysqld *Mysqld) detectFlavor() (string, error) { - - version, err := mysqld.getVersionString() - if err != nil { - return version, err - } - - // Check if the flavor has been defined - // This takes first precedence. - - if strings.Contains(version, "MySQL Community Server") { - return flavorMySQL, nil - } else if strings.Contains(version, "Percona") { - return flavorPercona, nil - } else if strings.Contains(version, "MariaDB") { - return flavorMariaDB, nil - } - - return "", fmt.Errorf("Could not autodetect flavor!") - -} - -func (mysqld *Mysqld) detectVersion() (serverVersion, error) { - - version, err := mysqld.getVersionString() - if err != nil { - return nil, err - } - - // Detect server version - re := regexp.MustCompile(`Ver [0-9]+\.[0-9]+\.[0-9]+`) - ver := re.Find([]byte(version))[4:] - re = regexp.MustCompile(`[0-9]+`) - v := re.FindAll([]byte(ver), -1) - major, _ := strconv.Atoi(string(v[0])) - minor, _ := strconv.Atoi(string(v[1])) - patch, _ := strconv.Atoi(string(v[2])) - - return serverVersion{major, minor, patch}, nil - -} diff --git a/go/vt/mysqlctl/capabilityset.go b/go/vt/mysqlctl/capabilityset.go new file mode 100644 index 00000000000..4650f1dc2ae --- /dev/null +++ b/go/vt/mysqlctl/capabilityset.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 The Vitess Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + Detect server flavors and capabilities +*/ + +package mysqlctl + +type mysqlFlavor string + +const ( + flavorMySQL mysqlFlavor = "mysql" + flavorPercona mysqlFlavor = "percona" + flavorMariaDB mysqlFlavor = "mariadb" +) + +// Mysqld is the object that represents a mysqld daemon running on this server. +type CapabilitySet struct { + mysqld *Mysqld +} + + +func NewCapabilitySet(mysqld *Mysqld) (c CapabilitySet) { + c.mysqld = mysqld + return +} + +func (c *CapabilitySet) HasMySQLUpgradeInServer() bool { + return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 8, Minor: 0, Patch: 16}) +} +func (c *CapabilitySet) HasInitializeInServer() bool { + return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 5, Minor: 7, Patch: 0}) +} +func (c *CapabilitySet) HasMySQLxEnabledByDefault() bool { + return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 8, Minor: 0, Patch: 11}) +} +func (c *CapabilitySet) HasPersistConfig() bool { + return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 8, Minor: 0, Patch: 0}) +} +func (c *CapabilitySet) HasShutdownCommand() bool { + return (c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 5, Minor: 7, Patch: 9})) || (c.IsMariaDB() && c.mysqld.version.greaterThan(serverVersion{Major: 10, Minor: 0, Patch: 4})) +} +func (c *CapabilitySet) HasBackupLocks() bool { + return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 8, Minor: 0, Patch: 0}) +} +func (c *CapabilitySet) HasDefaultUft8mb4() bool { + return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 8, Minor: 0, Patch: 0}) +} +func (c *CapabilitySet) HasSemiSyncEnabledByDefault() bool { + return c.IsMariaDB() && c.mysqld.version.greaterThan(serverVersion{Major: 10, Minor: 3, Patch: 3}) +} + +// IsMySQLLike tests if the server is either MySQL +// or Percona Server. At least currently, Vitess doesn't +// make use of any specific Percona Server features. +func (c *CapabilitySet) IsMySQLLike() bool { + return c.mysqld.flavor == flavorMySQL || c.mysqld.flavor == flavorPercona +} + +// IsMariaDB tests if the server is MariaDB. +// IsMySQLLike() and IsMariaDB() are mutually exclusive +func (c *CapabilitySet) IsMariaDB() bool { + return c.mysqld.flavor == flavorMariaDB +} diff --git a/go/vt/mysqlctl/cmd.go b/go/vt/mysqlctl/cmd.go index 95701b388d4..630d44904b3 100644 --- a/go/vt/mysqlctl/cmd.go +++ b/go/vt/mysqlctl/cmd.go @@ -51,7 +51,7 @@ func CreateMysqldAndMycnf(tabletUID uint32, mysqlSocket string, mysqlPort int32) return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err) } - mysqld, _ := NewMysqld(dbcfgs) + mysqld := NewMysqld(dbcfgs) return mysqld, mycnf, nil } @@ -70,6 +70,6 @@ func OpenMysqldAndMycnf(tabletUID uint32) (*Mysqld, *Mycnf, error) { return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err) } - mysqld, _ := NewMysqld(dbcfgs) + mysqld := NewMysqld(dbcfgs) return mysqld, mycnf, nil } diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 3688fcfb53b..dd536e63bb9 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -85,8 +85,9 @@ type Mysqld struct { dbaPool *dbconnpool.ConnectionPool appPool *dbconnpool.ConnectionPool - flavor string version serverVersion + flavor mysqlFlavor + c CapabilitySet // mutex protects the fields below. mutex sync.Mutex @@ -96,7 +97,7 @@ type Mysqld struct { // NewMysqld creates a Mysqld object based on the provided configuration // and connection parameters. -func NewMysqld(dbcfgs *dbconfigs.DBConfigs) (*Mysqld, error) { +func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { result := &Mysqld{ dbcfgs: dbcfgs, } @@ -109,19 +110,17 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) (*Mysqld, error) { result.appPool = dbconnpool.NewConnectionPool("AppConnPool", *appPoolSize, *appIdleTimeout, *poolDynamicHostnameResolution) result.appPool.Open(dbcfgs.AppWithDB(), appMysqlStats) - var err error - result.flavor, err = result.detectFlavor() - if err != nil { - return result, err - } - - result.version, err = result.detectVersion() - if err != nil { - return result, err - } + result.flavor, result.version = result.DetectFlavorVersion("version string here") + result.c = NewCapabilitySet(result) log.Infof("mysqld is flavor: %s, version: %v", result.flavor, result.version) - return result, nil + return result + +} + +// This becomes part of MySQLd! +func (mysqld *Mysqld) DetectFlavorVersion(verstring string) (mysqlFlavor, serverVersion) { + return flavorMySQL, serverVersion{Major: 8, Minor: 0, Patch: 16} } // RunMysqlUpgrade will run the mysql_upgrade program on the current @@ -139,7 +138,7 @@ func (mysqld *Mysqld) RunMysqlUpgrade() error { return client.RunMysqlUpgrade(context.TODO()) } - if mysqld.HasCapability(CapabilityMySQLUpgradeInServer) { + if mysqld.c.HasMySQLUpgradeInServer() { log.Warningf("MySQL version has built-in upgrade, skipping RunMySQLUpgrade") return nil } @@ -229,8 +228,9 @@ func (mysqld *Mysqld) startNoWait(ctx context.Context, cnf *Mycnf, mysqldArgs .. return err } - if mysqld.HasCapability(CapabilitySystemd) { - // The capabilities system did not detect mysqld_safe. + //mysqld.HasCapability(CapabilitySystemd) { + if true { + // The capabilities system did not detect mysqld_safe. // It is assumed that this mysqld package is systemd enabled. log.Info("launching mysqld instead of mysqld_safe") name, err = binaryPath(dir, "mysqld") @@ -580,7 +580,7 @@ func (mysqld *Mysqld) installDataDir(cnf *Mycnf) error { return err } - if mysqld.HasCapability(CapabilityInitializeInServer) { + if mysqld.c.HasMySQLUpgradeInServer() { log.Infof("Installing data dir with mysqld --initialize-insecure") args := []string{ "--defaults-file=" + cnf.path, @@ -667,7 +667,7 @@ func (mysqld *Mysqld) getMycnfTemplates(root string) []string { // Percona Server == MySQL in this context f := flavorMariaDB - if mysqld.IsMySQLLike() { + if mysqld.c.IsMySQLLike() { f = flavorMySQL } @@ -678,7 +678,7 @@ func (mysqld *Mysqld) getMycnfTemplates(root string) []string { } // master_{flavor}{major}{minor}.cnf - p = path.Join(root, fmt.Sprintf("config/mycnf/master_%s%d%d.cnf", f, mysqld.MajorVersion(), mysqld.MinorVersion())) + p = path.Join(root, fmt.Sprintf("config/mycnf/master_%s%d%d.cnf", f, mysqld.version.Major, mysqld.version.Minor)) _, err = os.Stat(p) if err == nil && !contains(cnfTemplatePaths, p) { cnfTemplatePaths = append(cnfTemplatePaths, p) diff --git a/go/vt/mysqlctl/version.go b/go/vt/mysqlctl/version.go new file mode 100644 index 00000000000..03005f454be --- /dev/null +++ b/go/vt/mysqlctl/version.go @@ -0,0 +1,37 @@ +/* +Copyright 2019 The Vitess Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + Detect server flavors and capabilities +*/ + +package mysqlctl + +type serverVersion struct { + Major, Minor, Patch int +} + +func (v *serverVersion) greaterThan(compare serverVersion) bool { + if v.Major > compare.Major { + return true + } else if v.Major == compare.Major && v.Minor > compare.Minor { + return true + } else if v.Major == compare.Major && v.Minor == compare.Minor && v.Patch >= compare.Patch { + return true + } else { + return false + } +} diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index a39abb28489..be2eb3070fe 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -104,7 +104,7 @@ func Init() (*Env, error) { } te.Dbcfgs = dbconfigs.NewTestDBConfigs(te.cluster.MySQLConnParams(), te.cluster.MySQLAppDebugConnParams(), te.cluster.DbName()) - te.Mysqld, _ = mysqlctl.NewMysqld(te.Dbcfgs) + te.Mysqld = mysqlctl.NewMysqld(te.Dbcfgs) te.SchemaEngine = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) te.SchemaEngine.InitDBConfig(te.Dbcfgs) From 469a94ec3673e7c5827e989213a286b0cd1df933 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 16 Jul 2019 10:57:35 -0700 Subject: [PATCH 04/18] Fixed mysql_safe to not use capabilitities Reverted small changes to configs Signed-off-by: Morgan Tocker --- config/mycnf/default.cnf | 2 +- config/mycnf/master_mysql57.cnf | 1 + go/vt/mysqlctl/cmd.go | 6 ++---- go/vt/mysqlctl/mysqld.go | 21 ++++++++------------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/config/mycnf/default.cnf b/config/mycnf/default.cnf index 2fe69d72def..6e02dc4de5f 100644 --- a/config/mycnf/default.cnf +++ b/config/mycnf/default.cnf @@ -2,7 +2,7 @@ sql_mode = STRICT_TRANS_TABLES back_log = 50 -binlog_format = row +binlog_format = statement character_set_server = utf8 collation_server = utf8_general_ci connect_timeout = 30 diff --git a/config/mycnf/master_mysql57.cnf b/config/mycnf/master_mysql57.cnf index 7c62eabc2d9..6743ffbe5e8 100644 --- a/config/mycnf/master_mysql57.cnf +++ b/config/mycnf/master_mysql57.cnf @@ -4,6 +4,7 @@ gtid_mode = ON log_bin log_slave_updates enforce_gtid_consistency +innodb_use_native_aio = 0 # Crash-safe replication settings. master_info_repository = TABLE diff --git a/go/vt/mysqlctl/cmd.go b/go/vt/mysqlctl/cmd.go index 630d44904b3..7289d999570 100644 --- a/go/vt/mysqlctl/cmd.go +++ b/go/vt/mysqlctl/cmd.go @@ -51,8 +51,7 @@ func CreateMysqldAndMycnf(tabletUID uint32, mysqlSocket string, mysqlPort int32) return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err) } - mysqld := NewMysqld(dbcfgs) - return mysqld, mycnf, nil + return NewMysqld(dbcfgs), mycnf, nil } // OpenMysqldAndMycnf returns a Mysqld and a Mycnf object to use for working with a MySQL @@ -70,6 +69,5 @@ func OpenMysqldAndMycnf(tabletUID uint32) (*Mysqld, *Mycnf, error) { return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err) } - mysqld := NewMysqld(dbcfgs) - return mysqld, mycnf, nil + return NewMysqld(dbcfgs), mycnf, nil } diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 3640c346a83..b0206c8e4de 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -87,7 +87,7 @@ type Mysqld struct { version serverVersion flavor mysqlFlavor - c CapabilitySet + c CapabilitySet // mutex protects the fields below. mutex sync.Mutex @@ -227,18 +227,13 @@ func (mysqld *Mysqld) startNoWait(ctx context.Context, cnf *Mycnf, mysqldArgs .. if err != nil { return err } - - //mysqld.HasCapability(CapabilitySystemd) { - if true { - // The capabilities system did not detect mysqld_safe. - // It is assumed that this mysqld package is systemd enabled. - log.Info("launching mysqld instead of mysqld_safe") + name, err = binaryPath(dir, "mysqld_safe") + if err != nil { + // The movement to use systemd means that mysqld_safe is not always provided. + // This should not be considered an issue do not generate a warning. + log.Infof("%v: trying to launch mysqld instead", err) name, err = binaryPath(dir, "mysqld") - if err != nil { - return err - } - } else { - name, err = binaryPath(dir, "mysqld_safe") + // If this also fails, return an error. if err != nil { return err } @@ -249,7 +244,7 @@ func (mysqld *Mysqld) startNoWait(ctx context.Context, cnf *Mycnf, mysqldArgs .. } arg := []string{ "--defaults-file=" + cnf.path, - "--basedir=" + mysqlBaseDir, + "--basedir" + mysqlBaseDir, } arg = append(arg, mysqldArgs...) env := []string{os.ExpandEnv("LD_LIBRARY_PATH=$VT_MYSQL_ROOT/lib/mysql")} From cd47164d5e2c796b49a3f5e0e9989dfae5e4448c Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 16 Jul 2019 12:51:46 -0700 Subject: [PATCH 05/18] fix version detection Signed-off-by: Morgan Tocker --- config/mycnf/default.cnf | 2 +- go/vt/mysqlctl/mysqld.go | 52 +++++++++++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/config/mycnf/default.cnf b/config/mycnf/default.cnf index 6e02dc4de5f..cb6711d3fa7 100644 --- a/config/mycnf/default.cnf +++ b/config/mycnf/default.cnf @@ -2,7 +2,7 @@ sql_mode = STRICT_TRANS_TABLES back_log = 50 -binlog_format = statement +binlog_format = statement character_set_server = utf8 collation_server = utf8_general_ci connect_timeout = 30 diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index b0206c8e4de..4de130565ff 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -35,6 +35,8 @@ import ( "os/exec" "path" "path/filepath" + "regexp" + "strconv" "strings" "sync" "time" @@ -110,7 +112,8 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { result.appPool = dbconnpool.NewConnectionPool("AppConnPool", *appPoolSize, *appIdleTimeout, *poolDynamicHostnameResolution) result.appPool.Open(dbcfgs.AppWithDB(), appMysqlStats) - result.flavor, result.version = result.DetectFlavorVersion("version string here") + version, _ := result.getVersionString() + result.flavor, result.version = result.DetectFlavorVersion(version) result.c = NewCapabilitySet(result) log.Infof("mysqld is flavor: %s, version: %v", result.flavor, result.version) @@ -118,9 +121,47 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { } -// This becomes part of MySQLd! -func (mysqld *Mysqld) DetectFlavorVersion(verstring string) (mysqlFlavor, serverVersion) { - return flavorMySQL, serverVersion{Major: 8, Minor: 0, Patch: 16} +func (mysqld *Mysqld) getVersionString() (string, error) { + + mysqlRoot, err := vtenv.VtMysqlRoot() + if err != nil { + return "", err + } + mysqldPath, err := binaryPath(mysqlRoot, "mysqld") + if err != nil { + return "", err + } + + _, version, err := execCmd(mysqldPath, []string{"--version"}, nil, mysqlRoot, nil) + if err != nil { + return "", err + } + + return version, nil + +} + +// DetectFlavorVersion takes the output of mysqld --version +// and parses it into a flavor and version +func (mysqld *Mysqld) DetectFlavorVersion(version string) (flavor mysqlFlavor, ver serverVersion) { + + if strings.Contains(version, "MySQL") { + flavor = flavorMySQL + } else if strings.Contains(version, "Percona") { + flavor = flavorPercona + } else if strings.Contains(version, "MariaDB") { + flavor = flavorMariaDB + } + re := regexp.MustCompile(`Ver ([0-9]+)\.([0-9]+)\.([0-9]+)`) + v := re.FindStringSubmatch(version) + if len(v) != 4 { + log.Errorf("Could not parse flavor and version from: %s", version) + } + ver.Major, _ = strconv.Atoi(string(v[1])) + ver.Minor, _ = strconv.Atoi(string(v[2])) + ver.Patch, _ = strconv.Atoi(string(v[3])) + + return } // RunMysqlUpgrade will run the mysql_upgrade program on the current @@ -572,8 +613,7 @@ func (mysqld *Mysqld) installDataDir(cnf *Mycnf) error { if err != nil { return err } - - if mysqld.c.HasMySQLUpgradeInServer() { + if mysqld.c.HasInitializeInServer() { log.Infof("Installing data dir with mysqld --initialize-insecure") args := []string{ "--defaults-file=" + cnf.path, From e1ff1b7618b35485dab0044edb419d437cc0b576 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 16 Jul 2019 14:10:21 -0700 Subject: [PATCH 06/18] Fix style, re-add EXTRA_MY_CNF from review Signed-off-by: Morgan Tocker --- examples/local/vttablet-up.sh | 2 ++ go/vt/mysqlctl/version.go | 21 +++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/examples/local/vttablet-up.sh b/examples/local/vttablet-up.sh index 013b657942f..13169c79455 100755 --- a/examples/local/vttablet-up.sh +++ b/examples/local/vttablet-up.sh @@ -41,6 +41,8 @@ init_db_sql_file="$VTROOT/config/init_db.sql" # Previously this file set EXTRA_MY_CNF based on MYSQL_FLAVOR # and enabled RBR. It now relies on mysqlctl to autodetect +export EXTRA_MY_CNF=$VTROOT/config/mycnf/default-fast.cnf:$VTROOT/config/mycnf/rbr.cnf + mkdir -p $VTDATAROOT/backups # Start 3 vttablets by default. diff --git a/go/vt/mysqlctl/version.go b/go/vt/mysqlctl/version.go index 03005f454be..155c556f23e 100644 --- a/go/vt/mysqlctl/version.go +++ b/go/vt/mysqlctl/version.go @@ -21,17 +21,18 @@ limitations under the License. package mysqlctl type serverVersion struct { - Major, Minor, Patch int + Major, Minor, Patch int } func (v *serverVersion) greaterThan(compare serverVersion) bool { - if v.Major > compare.Major { - return true - } else if v.Major == compare.Major && v.Minor > compare.Minor { - return true - } else if v.Major == compare.Major && v.Minor == compare.Minor && v.Patch >= compare.Patch { - return true - } else { - return false - } + if v.Major > compare.Major { + return true + } + if v.Major == compare.Major && v.Minor > compare.Minor { + return true + } + if v.Major == compare.Major && v.Minor == compare.Minor && v.Patch >= compare.Patch { + return true + } + return false } From ed86bd53096f1f0ee3897244e3078fa197f8afd8 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 16 Jul 2019 14:48:16 -0700 Subject: [PATCH 07/18] Address review feedback Signed-off-by: Morgan Tocker --- go/vt/mysqlctl/mysqld.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 4de130565ff..6643c9a16ee 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -79,6 +79,8 @@ var ( dbaMysqlStats = stats.NewTimings("MysqlDba", "MySQL DBA stats", "operation") allprivsMysqlStats = stats.NewTimings("MysqlAllPrivs", "MySQl Stats for all privs", "operation") appMysqlStats = stats.NewTimings("MysqlApp", "MySQL app stats", "operation") + + versionRegex = regexp.MustCompile(`Ver ([0-9]+)\.([0-9]+)\.([0-9]+)`) ) // Mysqld is the object that represents a mysqld daemon running on this server. @@ -152,8 +154,7 @@ func (mysqld *Mysqld) DetectFlavorVersion(version string) (flavor mysqlFlavor, v } else if strings.Contains(version, "MariaDB") { flavor = flavorMariaDB } - re := regexp.MustCompile(`Ver ([0-9]+)\.([0-9]+)\.([0-9]+)`) - v := re.FindStringSubmatch(version) + v := versionRegex.FindStringSubmatch(version) if len(v) != 4 { log.Errorf("Could not parse flavor and version from: %s", version) } From 79dbc8e6a192f819dc114966cfe678113fc60644 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 16 Jul 2019 15:05:38 -0700 Subject: [PATCH 08/18] address remaining feedback Signed-off-by: Morgan Tocker --- go/vt/mysqlctl/capabilityset.go | 16 ++++++++-------- go/vt/mysqlctl/mysqld.go | 10 +++------- go/vt/mysqlctl/version.go | 2 +- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/go/vt/mysqlctl/capabilityset.go b/go/vt/mysqlctl/capabilityset.go index 4650f1dc2ae..54d65d52455 100644 --- a/go/vt/mysqlctl/capabilityset.go +++ b/go/vt/mysqlctl/capabilityset.go @@ -40,28 +40,28 @@ func NewCapabilitySet(mysqld *Mysqld) (c CapabilitySet) { } func (c *CapabilitySet) HasMySQLUpgradeInServer() bool { - return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 8, Minor: 0, Patch: 16}) + return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 16}) } func (c *CapabilitySet) HasInitializeInServer() bool { - return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 5, Minor: 7, Patch: 0}) + return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 5, Minor: 7, Patch: 0}) } func (c *CapabilitySet) HasMySQLxEnabledByDefault() bool { - return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 8, Minor: 0, Patch: 11}) + return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 11}) } func (c *CapabilitySet) HasPersistConfig() bool { - return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 8, Minor: 0, Patch: 0}) + return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 0}) } func (c *CapabilitySet) HasShutdownCommand() bool { - return (c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 5, Minor: 7, Patch: 9})) || (c.IsMariaDB() && c.mysqld.version.greaterThan(serverVersion{Major: 10, Minor: 0, Patch: 4})) + return (c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 5, Minor: 7, Patch: 9})) || (c.IsMariaDB() && c.mysqld.version.atLeast(serverVersion{Major: 10, Minor: 0, Patch: 4})) } func (c *CapabilitySet) HasBackupLocks() bool { - return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 8, Minor: 0, Patch: 0}) + return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 0}) } func (c *CapabilitySet) HasDefaultUft8mb4() bool { - return c.IsMySQLLike() && c.mysqld.version.greaterThan(serverVersion{Major: 8, Minor: 0, Patch: 0}) + return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 0}) } func (c *CapabilitySet) HasSemiSyncEnabledByDefault() bool { - return c.IsMariaDB() && c.mysqld.version.greaterThan(serverVersion{Major: 10, Minor: 3, Patch: 3}) + return c.IsMariaDB() && c.mysqld.version.atLeast(serverVersion{Major: 10, Minor: 3, Patch: 3}) } // IsMySQLLike tests if the server is either MySQL diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 6643c9a16ee..110c4034f2f 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -120,11 +120,9 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { log.Infof("mysqld is flavor: %s, version: %v", result.flavor, result.version) return result - } func (mysqld *Mysqld) getVersionString() (string, error) { - mysqlRoot, err := vtenv.VtMysqlRoot() if err != nil { return "", err @@ -133,30 +131,28 @@ func (mysqld *Mysqld) getVersionString() (string, error) { if err != nil { return "", err } - _, version, err := execCmd(mysqldPath, []string{"--version"}, nil, mysqlRoot, nil) if err != nil { return "", err } - return version, nil - } // DetectFlavorVersion takes the output of mysqld --version // and parses it into a flavor and version func (mysqld *Mysqld) DetectFlavorVersion(version string) (flavor mysqlFlavor, ver serverVersion) { - if strings.Contains(version, "MySQL") { flavor = flavorMySQL } else if strings.Contains(version, "Percona") { flavor = flavorPercona } else if strings.Contains(version, "MariaDB") { flavor = flavorMariaDB + } else { + log.Fatalf("unrecognized server flavor from: %s", version) } v := versionRegex.FindStringSubmatch(version) if len(v) != 4 { - log.Errorf("Could not parse flavor and version from: %s", version) + log.Fatalf("Could not parse server version from: %s", version) } ver.Major, _ = strconv.Atoi(string(v[1])) ver.Minor, _ = strconv.Atoi(string(v[2])) diff --git a/go/vt/mysqlctl/version.go b/go/vt/mysqlctl/version.go index 155c556f23e..05bb18c20ef 100644 --- a/go/vt/mysqlctl/version.go +++ b/go/vt/mysqlctl/version.go @@ -24,7 +24,7 @@ type serverVersion struct { Major, Minor, Patch int } -func (v *serverVersion) greaterThan(compare serverVersion) bool { +func (v *serverVersion) atLeast(compare serverVersion) bool { if v.Major > compare.Major { return true } From a02bbad4113590cd8687f25af5ee7b860346375e Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 16 Jul 2019 15:07:53 -0700 Subject: [PATCH 09/18] Remove rogue added space Signed-off-by: Morgan Tocker --- go/vt/mysqlctl/mysqld.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 110c4034f2f..970edb08010 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -676,7 +676,6 @@ func contains(haystack []string, needle string) bool { } func (mysqld *Mysqld) getMycnfTemplates(root string) []string { - if *mycnfTemplateFile != "" { return []string{*mycnfTemplateFile} } From 5339cbb5206679ea8847d319b7011e15e05653a4 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Mon, 29 Jul 2019 10:13:33 -0700 Subject: [PATCH 10/18] Address PR feedback Signed-off-by: Morgan Tocker --- examples/local/vttablet-up.sh | 2 +- go/cmd/vtcombo/main.go | 6 ++- go/cmd/vttablet/vttablet.go | 5 ++- go/vt/mysqlctl/capabilityset.go | 29 ++++++++------- go/vt/mysqlctl/cmd.go | 12 +++++- go/vt/mysqlctl/mysqld.go | 37 +++++++++---------- .../tabletserver/vstreamer/testenv/testenv.go | 6 ++- 7 files changed, 58 insertions(+), 39 deletions(-) diff --git a/examples/local/vttablet-up.sh b/examples/local/vttablet-up.sh index 13169c79455..75c3b191d04 100755 --- a/examples/local/vttablet-up.sh +++ b/examples/local/vttablet-up.sh @@ -39,7 +39,7 @@ source $script_root/env.sh init_db_sql_file="$VTROOT/config/init_db.sql" # Previously this file set EXTRA_MY_CNF based on MYSQL_FLAVOR -# and enabled RBR. It now relies on mysqlctl to autodetect +# It now relies on mysqlctl to autodetect export EXTRA_MY_CNF=$VTROOT/config/mycnf/default-fast.cnf:$VTROOT/config/mycnf/rbr.cnf diff --git a/go/cmd/vtcombo/main.go b/go/cmd/vtcombo/main.go index f92181861f4..39f0f57dbe5 100644 --- a/go/cmd/vtcombo/main.go +++ b/go/cmd/vtcombo/main.go @@ -99,7 +99,11 @@ func main() { if err != nil { log.Warning(err) } - mysqld := mysqlctl.NewMysqld(dbcfgs) + mysqld, err := mysqlctl.NewMysqld(dbcfgs) + if err != nil { + log.Errorf("Creating mysqld failed: %v", err) + exit.Return(1) + } servenv.OnClose(mysqld.Close) // tablets configuration and init. diff --git a/go/cmd/vttablet/vttablet.go b/go/cmd/vttablet/vttablet.go index 76493fc881a..18a109c8265 100644 --- a/go/cmd/vttablet/vttablet.go +++ b/go/cmd/vttablet/vttablet.go @@ -117,7 +117,10 @@ func main() { // Create mysqld and register the health reporter (needs to be done // before initializing the agent, so the initial health check // done by the agent has the right reporter) - mysqld := mysqlctl.NewMysqld(dbcfgs) + mysqld, err := mysqlctl.NewMysqld(dbcfgs) + if err != nil { + log.Exitf("NewActionAgent() failed: %v", err) + } servenv.OnClose(mysqld.Close) // Depends on both query and updateStream. diff --git a/go/vt/mysqlctl/capabilityset.go b/go/vt/mysqlctl/capabilityset.go index 54d65d52455..568c75bd096 100644 --- a/go/vt/mysqlctl/capabilityset.go +++ b/go/vt/mysqlctl/capabilityset.go @@ -30,49 +30,50 @@ const ( // Mysqld is the object that represents a mysqld daemon running on this server. type CapabilitySet struct { - mysqld *Mysqld + flavor mysqlFlavor + version serverVersion } - -func NewCapabilitySet(mysqld *Mysqld) (c CapabilitySet) { - c.mysqld = mysqld +func NewCapabilitySet(f mysqlFlavor, v serverVersion) (c CapabilitySet) { + c.flavor = f + c.version = v return } func (c *CapabilitySet) HasMySQLUpgradeInServer() bool { - return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 16}) + return c.IsMySQLLike() && c.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 16}) } func (c *CapabilitySet) HasInitializeInServer() bool { - return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 5, Minor: 7, Patch: 0}) + return c.IsMySQLLike() && c.version.atLeast(serverVersion{Major: 5, Minor: 7, Patch: 0}) } func (c *CapabilitySet) HasMySQLxEnabledByDefault() bool { - return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 11}) + return c.IsMySQLLike() && c.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 11}) } func (c *CapabilitySet) HasPersistConfig() bool { - return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 0}) + return c.IsMySQLLike() && c.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 0}) } func (c *CapabilitySet) HasShutdownCommand() bool { - return (c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 5, Minor: 7, Patch: 9})) || (c.IsMariaDB() && c.mysqld.version.atLeast(serverVersion{Major: 10, Minor: 0, Patch: 4})) + return (c.IsMySQLLike() && c.version.atLeast(serverVersion{Major: 5, Minor: 7, Patch: 9})) || (c.IsMariaDB() && c.version.atLeast(serverVersion{Major: 10, Minor: 0, Patch: 4})) } func (c *CapabilitySet) HasBackupLocks() bool { - return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 0}) + return c.IsMySQLLike() && c.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 0}) } func (c *CapabilitySet) HasDefaultUft8mb4() bool { - return c.IsMySQLLike() && c.mysqld.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 0}) + return c.IsMySQLLike() && c.version.atLeast(serverVersion{Major: 8, Minor: 0, Patch: 0}) } func (c *CapabilitySet) HasSemiSyncEnabledByDefault() bool { - return c.IsMariaDB() && c.mysqld.version.atLeast(serverVersion{Major: 10, Minor: 3, Patch: 3}) + return c.IsMariaDB() && c.version.atLeast(serverVersion{Major: 10, Minor: 3, Patch: 3}) } // IsMySQLLike tests if the server is either MySQL // or Percona Server. At least currently, Vitess doesn't // make use of any specific Percona Server features. func (c *CapabilitySet) IsMySQLLike() bool { - return c.mysqld.flavor == flavorMySQL || c.mysqld.flavor == flavorPercona + return c.flavor == flavorMySQL || c.flavor == flavorPercona } // IsMariaDB tests if the server is MariaDB. // IsMySQLLike() and IsMariaDB() are mutually exclusive func (c *CapabilitySet) IsMariaDB() bool { - return c.mysqld.flavor == flavorMariaDB + return c.flavor == flavorMariaDB } diff --git a/go/vt/mysqlctl/cmd.go b/go/vt/mysqlctl/cmd.go index 7289d999570..1a2240de471 100644 --- a/go/vt/mysqlctl/cmd.go +++ b/go/vt/mysqlctl/cmd.go @@ -51,7 +51,11 @@ func CreateMysqldAndMycnf(tabletUID uint32, mysqlSocket string, mysqlPort int32) return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err) } - return NewMysqld(dbcfgs), mycnf, nil + mysqld, err := NewMysqld(dbcfgs) + if err != nil { + return nil, nil, err + } + return mysqld, mycnf, nil } // OpenMysqldAndMycnf returns a Mysqld and a Mycnf object to use for working with a MySQL @@ -69,5 +73,9 @@ func OpenMysqldAndMycnf(tabletUID uint32) (*Mysqld, *Mycnf, error) { return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err) } - return NewMysqld(dbcfgs), mycnf, nil + mysqld, err := NewMysqld(dbcfgs) + if err != nil { + return nil, nil, err + } + return mysqld, mycnf, nil } diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 970edb08010..1ee7a3f1a57 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -89,9 +89,7 @@ type Mysqld struct { dbaPool *dbconnpool.ConnectionPool appPool *dbconnpool.ConnectionPool - version serverVersion - flavor mysqlFlavor - c CapabilitySet + capabilities CapabilitySet // mutex protects the fields below. mutex sync.Mutex @@ -101,7 +99,7 @@ type Mysqld struct { // NewMysqld creates a Mysqld object based on the provided configuration // and connection parameters. -func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { +func NewMysqld(dbcfgs *dbconfigs.DBConfigs) (*Mysqld, error) { result := &Mysqld{ dbcfgs: dbcfgs, } @@ -114,15 +112,17 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { result.appPool = dbconnpool.NewConnectionPool("AppConnPool", *appPoolSize, *appIdleTimeout, *poolDynamicHostnameResolution) result.appPool.Open(dbcfgs.AppWithDB(), appMysqlStats) - version, _ := result.getVersionString() - result.flavor, result.version = result.DetectFlavorVersion(version) - result.c = NewCapabilitySet(result) + version, _ := getVersionString() + f, v, err := parseVersionString(version) + if err != nil { + return nil, err + } - log.Infof("mysqld is flavor: %s, version: %v", result.flavor, result.version) - return result + result.capabilities = NewCapabilitySet(f, v) + return result, nil } -func (mysqld *Mysqld) getVersionString() (string, error) { +func getVersionString() (string, error) { mysqlRoot, err := vtenv.VtMysqlRoot() if err != nil { return "", err @@ -138,9 +138,8 @@ func (mysqld *Mysqld) getVersionString() (string, error) { return version, nil } -// DetectFlavorVersion takes the output of mysqld --version -// and parses it into a flavor and version -func (mysqld *Mysqld) DetectFlavorVersion(version string) (flavor mysqlFlavor, ver serverVersion) { +// parse the output of mysqld --version into a flavor and version +func parseVersionString(version string) (flavor mysqlFlavor, ver serverVersion, err error) { if strings.Contains(version, "MySQL") { flavor = flavorMySQL } else if strings.Contains(version, "Percona") { @@ -148,11 +147,11 @@ func (mysqld *Mysqld) DetectFlavorVersion(version string) (flavor mysqlFlavor, v } else if strings.Contains(version, "MariaDB") { flavor = flavorMariaDB } else { - log.Fatalf("unrecognized server flavor from: %s", version) + return flavor, ver, fmt.Errorf("unrecognized server flavor from: %s", version) } v := versionRegex.FindStringSubmatch(version) if len(v) != 4 { - log.Fatalf("Could not parse server version from: %s", version) + return flavor, ver, fmt.Errorf("Could not parse server version from: %s", version) } ver.Major, _ = strconv.Atoi(string(v[1])) ver.Minor, _ = strconv.Atoi(string(v[2])) @@ -176,7 +175,7 @@ func (mysqld *Mysqld) RunMysqlUpgrade() error { return client.RunMysqlUpgrade(context.TODO()) } - if mysqld.c.HasMySQLUpgradeInServer() { + if mysqld.capabilities.HasMySQLUpgradeInServer() { log.Warningf("MySQL version has built-in upgrade, skipping RunMySQLUpgrade") return nil } @@ -610,7 +609,7 @@ func (mysqld *Mysqld) installDataDir(cnf *Mycnf) error { if err != nil { return err } - if mysqld.c.HasInitializeInServer() { + if mysqld.capabilities.HasInitializeInServer() { log.Infof("Installing data dir with mysqld --initialize-insecure") args := []string{ "--defaults-file=" + cnf.path, @@ -696,7 +695,7 @@ func (mysqld *Mysqld) getMycnfTemplates(root string) []string { // Percona Server == MySQL in this context f := flavorMariaDB - if mysqld.c.IsMySQLLike() { + if mysqld.capabilities.IsMySQLLike() { f = flavorMySQL } @@ -707,7 +706,7 @@ func (mysqld *Mysqld) getMycnfTemplates(root string) []string { } // master_{flavor}{major}{minor}.cnf - p = path.Join(root, fmt.Sprintf("config/mycnf/master_%s%d%d.cnf", f, mysqld.version.Major, mysqld.version.Minor)) + p = path.Join(root, fmt.Sprintf("config/mycnf/master_%s%d%d.cnf", f, mysqld.capabilities.version.Major, mysqld.capabilities.version.Minor)) _, err = os.Stat(p) if err == nil && !contains(cnfTemplatePaths, p) { cnfTemplatePaths = append(cnfTemplatePaths, p) diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index be2eb3070fe..33b5852cbd8 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -104,7 +104,11 @@ func Init() (*Env, error) { } te.Dbcfgs = dbconfigs.NewTestDBConfigs(te.cluster.MySQLConnParams(), te.cluster.MySQLAppDebugConnParams(), te.cluster.DbName()) - te.Mysqld = mysqlctl.NewMysqld(te.Dbcfgs) + var err error + te.Mysqld, err = mysqlctl.NewMysqld(te.Dbcfgs) + if err != nil { + return nil, err + } te.SchemaEngine = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) te.SchemaEngine.InitDBConfig(te.Dbcfgs) From f88173553be4eae6632d544f7a082a7e7184c5f0 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Mon, 29 Jul 2019 10:17:59 -0700 Subject: [PATCH 11/18] Address PR feedback Signed-off-by: Morgan Tocker --- config/mycnf/master_mariadb.cnf | 2 ++ config/mycnf/master_mariadb100.cnf | 2 ++ config/mycnf/master_mariadb101.cnf | 2 ++ config/mycnf/master_mariadb102.cnf | 2 ++ config/mycnf/master_mariadb103.cnf | 2 ++ config/mycnf/master_mysql56.cnf | 2 ++ config/mycnf/master_mysql57.cnf | 2 ++ config/mycnf/master_mysql80.cnf | 2 ++ 8 files changed, 16 insertions(+) diff --git a/config/mycnf/master_mariadb.cnf b/config/mycnf/master_mariadb.cnf index 064e1307964..1e41cd8f3ce 100644 --- a/config/mycnf/master_mariadb.cnf +++ b/config/mycnf/master_mariadb.cnf @@ -1,3 +1,5 @@ +# This file is auto-included when MariaDB (any version) is detected. + # enable strict mode so it's safe to compare sequence numbers across different server IDs. gtid_strict_mode = 1 innodb_stats_persistent = 0 diff --git a/config/mycnf/master_mariadb100.cnf b/config/mycnf/master_mariadb100.cnf index 9f61d14d4be..ce85c641c13 100644 --- a/config/mycnf/master_mariadb100.cnf +++ b/config/mycnf/master_mariadb100.cnf @@ -1,3 +1,5 @@ +# This file is auto-included when MariaDB 10.0 is detected. + innodb_support_xa = 0 # Semi-sync replication is required for automated unplanned failover diff --git a/config/mycnf/master_mariadb101.cnf b/config/mycnf/master_mariadb101.cnf index 9f61d14d4be..d613b155d68 100644 --- a/config/mycnf/master_mariadb101.cnf +++ b/config/mycnf/master_mariadb101.cnf @@ -1,3 +1,5 @@ +# This file is auto-included when MariaDB 10.1 is detected. + innodb_support_xa = 0 # Semi-sync replication is required for automated unplanned failover diff --git a/config/mycnf/master_mariadb102.cnf b/config/mycnf/master_mariadb102.cnf index 9f61d14d4be..487baa9bf87 100644 --- a/config/mycnf/master_mariadb102.cnf +++ b/config/mycnf/master_mariadb102.cnf @@ -1,3 +1,5 @@ +# This file is auto-included when MariaDB 10.2 is detected. + innodb_support_xa = 0 # Semi-sync replication is required for automated unplanned failover diff --git a/config/mycnf/master_mariadb103.cnf b/config/mycnf/master_mariadb103.cnf index a4dfb50ab18..ac8b38404fd 100644 --- a/config/mycnf/master_mariadb103.cnf +++ b/config/mycnf/master_mariadb103.cnf @@ -1,3 +1,5 @@ +# This file is auto-included when MariaDB 10.3 is detected. + # enable strict mode so it's safe to compare sequence numbers across different server IDs. gtid_strict_mode = 1 innodb_stats_persistent = 0 diff --git a/config/mycnf/master_mysql56.cnf b/config/mycnf/master_mysql56.cnf index 2c802b2332a..dcb8a4e113f 100644 --- a/config/mycnf/master_mysql56.cnf +++ b/config/mycnf/master_mysql56.cnf @@ -1,3 +1,5 @@ +# This file is auto-included when MySQL 5.6 is detected. + # Options for enabling GTID # https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html gtid_mode = ON diff --git a/config/mycnf/master_mysql57.cnf b/config/mycnf/master_mysql57.cnf index 6743ffbe5e8..381b05ac14c 100644 --- a/config/mycnf/master_mysql57.cnf +++ b/config/mycnf/master_mysql57.cnf @@ -1,3 +1,5 @@ +# This file is auto-included when MySQL 5.7 is detected. + # Options for enabling GTID # https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html gtid_mode = ON diff --git a/config/mycnf/master_mysql80.cnf b/config/mycnf/master_mysql80.cnf index f81761ad906..e92b794ef9b 100644 --- a/config/mycnf/master_mysql80.cnf +++ b/config/mycnf/master_mysql80.cnf @@ -1,3 +1,5 @@ +# This file is auto-included when MySQL 8.0 is detected. + # Options for enabling GTID # https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html gtid_mode = ON From 2fde7d43bf6123e7fc865c73599dc5519fd5f856 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Mon, 29 Jul 2019 12:20:51 -0700 Subject: [PATCH 12/18] Fix broken test Signed-off-by: Morgan Tocker --- go/vt/mysqlctl/mycnf_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go/vt/mysqlctl/mycnf_test.go b/go/vt/mysqlctl/mycnf_test.go index 0298b7f8713..97d9c245170 100644 --- a/go/vt/mysqlctl/mycnf_test.go +++ b/go/vt/mysqlctl/mycnf_test.go @@ -106,7 +106,10 @@ func NoTestMycnfHook(t *testing.T) { os.Setenv("MY_VAR", "myvalue") dbcfgs, _ := dbconfigs.Init(cnf.SocketFile) - mysqld := NewMysqld(dbcfgs) + mysqld, err := NewMysqld(dbcfgs) + if err != nil { + t.Errorf("err: %v", err) + } servenv.OnClose(mysqld.Close) err := mysqld.InitConfig(cnf) From 555f55910e635670e42f0002c899848be42d2851 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Mon, 29 Jul 2019 14:06:15 -0700 Subject: [PATCH 13/18] Fixed tests Signed-off-by: Morgan Tocker --- go/vt/mysqlctl/mycnf_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/mysqlctl/mycnf_test.go b/go/vt/mysqlctl/mycnf_test.go index 97d9c245170..fdc73a3e7ae 100644 --- a/go/vt/mysqlctl/mycnf_test.go +++ b/go/vt/mysqlctl/mycnf_test.go @@ -112,7 +112,7 @@ func NoTestMycnfHook(t *testing.T) { } servenv.OnClose(mysqld.Close) - err := mysqld.InitConfig(cnf) + err = mysqld.InitConfig(cnf) if err != nil { t.Errorf("err: %v", err) } From e7f74749daf52b089489c446eeead22fc00e1469 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 6 Aug 2019 10:23:38 -0700 Subject: [PATCH 14/18] Addressed PR feedback Added tests for version detection Signed-off-by: Morgan Tocker --- go/cmd/vtcombo/main.go | 6 +- go/cmd/vttablet/vttablet.go | 5 +- go/vt/mysqlctl/cmd.go | 12 +- go/vt/mysqlctl/mycnf_test.go | 7 +- go/vt/mysqlctl/mysqld.go | 61 ++++++-- go/vt/mysqlctl/mysqld_test.go | 147 ++++++++++++++++++ .../tabletserver/vstreamer/testenv/testenv.go | 6 +- 7 files changed, 205 insertions(+), 39 deletions(-) create mode 100644 go/vt/mysqlctl/mysqld_test.go diff --git a/go/cmd/vtcombo/main.go b/go/cmd/vtcombo/main.go index 39f0f57dbe5..f92181861f4 100644 --- a/go/cmd/vtcombo/main.go +++ b/go/cmd/vtcombo/main.go @@ -99,11 +99,7 @@ func main() { if err != nil { log.Warning(err) } - mysqld, err := mysqlctl.NewMysqld(dbcfgs) - if err != nil { - log.Errorf("Creating mysqld failed: %v", err) - exit.Return(1) - } + mysqld := mysqlctl.NewMysqld(dbcfgs) servenv.OnClose(mysqld.Close) // tablets configuration and init. diff --git a/go/cmd/vttablet/vttablet.go b/go/cmd/vttablet/vttablet.go index 18a109c8265..76493fc881a 100644 --- a/go/cmd/vttablet/vttablet.go +++ b/go/cmd/vttablet/vttablet.go @@ -117,10 +117,7 @@ func main() { // Create mysqld and register the health reporter (needs to be done // before initializing the agent, so the initial health check // done by the agent has the right reporter) - mysqld, err := mysqlctl.NewMysqld(dbcfgs) - if err != nil { - log.Exitf("NewActionAgent() failed: %v", err) - } + mysqld := mysqlctl.NewMysqld(dbcfgs) servenv.OnClose(mysqld.Close) // Depends on both query and updateStream. diff --git a/go/vt/mysqlctl/cmd.go b/go/vt/mysqlctl/cmd.go index 1a2240de471..7289d999570 100644 --- a/go/vt/mysqlctl/cmd.go +++ b/go/vt/mysqlctl/cmd.go @@ -51,11 +51,7 @@ func CreateMysqldAndMycnf(tabletUID uint32, mysqlSocket string, mysqlPort int32) return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err) } - mysqld, err := NewMysqld(dbcfgs) - if err != nil { - return nil, nil, err - } - return mysqld, mycnf, nil + return NewMysqld(dbcfgs), mycnf, nil } // OpenMysqldAndMycnf returns a Mysqld and a Mycnf object to use for working with a MySQL @@ -73,9 +69,5 @@ func OpenMysqldAndMycnf(tabletUID uint32) (*Mysqld, *Mycnf, error) { return nil, nil, fmt.Errorf("couldn't Init dbconfigs: %v", err) } - mysqld, err := NewMysqld(dbcfgs) - if err != nil { - return nil, nil, err - } - return mysqld, mycnf, nil + return NewMysqld(dbcfgs), mycnf, nil } diff --git a/go/vt/mysqlctl/mycnf_test.go b/go/vt/mysqlctl/mycnf_test.go index fdc73a3e7ae..0298b7f8713 100644 --- a/go/vt/mysqlctl/mycnf_test.go +++ b/go/vt/mysqlctl/mycnf_test.go @@ -106,13 +106,10 @@ func NoTestMycnfHook(t *testing.T) { os.Setenv("MY_VAR", "myvalue") dbcfgs, _ := dbconfigs.Init(cnf.SocketFile) - mysqld, err := NewMysqld(dbcfgs) - if err != nil { - t.Errorf("err: %v", err) - } + mysqld := NewMysqld(dbcfgs) servenv.OnClose(mysqld.Close) - err = mysqld.InitConfig(cnf) + err := mysqld.InitConfig(cnf) if err != nil { t.Errorf("err: %v", err) } diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 1ee7a3f1a57..2a9d5afd347 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -99,7 +99,7 @@ type Mysqld struct { // NewMysqld creates a Mysqld object based on the provided configuration // and connection parameters. -func NewMysqld(dbcfgs *dbconfigs.DBConfigs) (*Mysqld, error) { +func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { result := &Mysqld{ dbcfgs: dbcfgs, } @@ -114,12 +114,43 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) (*Mysqld, error) { version, _ := getVersionString() f, v, err := parseVersionString(version) + + // Fallback if required if err != nil { - return nil, err + f, v, err = getVersionFromEnv() + if err != nil { + panic("Could not detect version from mysqld --version or MYSQL_FLAVOR") + } + } + // Unset ENV to make sure there is no split brain between legacy + // MySQL flavor and capabilities + os.Unsetenv("MYSQL_FLAVOR") + log.Infof("Using flavor: %v, version: %v", f, v) result.capabilities = NewCapabilitySet(f, v) - return result, nil + return result +} + +func getVersionFromEnv() (flavor mysqlFlavor, ver serverVersion, err error) { + + env := os.Getenv("MYSQL_FLAVOR") + + if env == "MariaDB" { + return flavorMariaDB, serverVersion{10, 0, 10}, nil + } + if env == "MariaDB103" { + return flavorMariaDB, serverVersion{10, 3, 7}, nil + } + if env == "MySQL80" { + return flavorMySQL, serverVersion{8, 0, 11}, nil + } + if env == "MySQL56" { + return flavorMySQL, serverVersion{5, 7, 10}, nil + } + + return flavor, ver, fmt.Errorf("Could not determine version from MYSQL_FLAVOR: %s", env) + } func getVersionString() (string, error) { @@ -140,22 +171,32 @@ func getVersionString() (string, error) { // parse the output of mysqld --version into a flavor and version func parseVersionString(version string) (flavor mysqlFlavor, ver serverVersion, err error) { - if strings.Contains(version, "MySQL") { - flavor = flavorMySQL - } else if strings.Contains(version, "Percona") { + + if strings.Contains(version, "Percona") { flavor = flavorPercona } else if strings.Contains(version, "MariaDB") { flavor = flavorMariaDB } else { - return flavor, ver, fmt.Errorf("unrecognized server flavor from: %s", version) + // OS distributed MySQL releases have a version string like: + // mysqld Ver 5.7.27-0ubuntu0.19.04.1 for Linux on x86_64 ((Ubuntu)) + flavor = flavorMySQL } v := versionRegex.FindStringSubmatch(version) if len(v) != 4 { return flavor, ver, fmt.Errorf("Could not parse server version from: %s", version) } - ver.Major, _ = strconv.Atoi(string(v[1])) - ver.Minor, _ = strconv.Atoi(string(v[2])) - ver.Patch, _ = strconv.Atoi(string(v[3])) + ver.Major, err = strconv.Atoi(string(v[1])) + if err != nil { + return flavor, ver, fmt.Errorf("Could not parse server version from: %s", version) + } + ver.Minor, err = strconv.Atoi(string(v[2])) + if err != nil { + return flavor, ver, fmt.Errorf("Could not parse server version from: %s", version) + } + ver.Patch, err = strconv.Atoi(string(v[3])) + if err != nil { + return flavor, ver, fmt.Errorf("Could not parse server version from: %s", version) + } return } diff --git a/go/vt/mysqlctl/mysqld_test.go b/go/vt/mysqlctl/mysqld_test.go new file mode 100644 index 00000000000..12539b878e2 --- /dev/null +++ b/go/vt/mysqlctl/mysqld_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2017 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysqlctl + +import ( + "os" + "testing" + // "vitess.io/vitess/go/vt/env" +) + +type testcase struct { + versionString string + version serverVersion + flavor mysqlFlavor +} + +func TestParseVersionString(t *testing.T) { + + var testcases = []testcase{ + + { + versionString: "mysqld Ver 5.7.27-0ubuntu0.19.04.1 for Linux on x86_64 ((Ubuntu))", + version: serverVersion{5, 7, 27}, + flavor: flavorMySQL, + }, + { + versionString: "mysqld Ver 5.6.43 for linux-glibc2.12 on x86_64 (MySQL Community Server (GPL))", + version: serverVersion{5, 6, 43}, + flavor: flavorMySQL, + }, + { + versionString: "mysqld Ver 5.7.26 for linux-glibc2.12 on x86_64 (MySQL Community Server (GPL))", + version: serverVersion{5, 7, 26}, + flavor: flavorMySQL, + }, + { + versionString: "mysqld Ver 8.0.16 for linux-glibc2.12 on x86_64 (MySQL Community Server - GPL)", + version: serverVersion{8, 0, 16}, + flavor: flavorMySQL, + }, + { + versionString: "mysqld Ver 5.7.26-29 for Linux on x86_64 (Percona Server (GPL), Release 29, Revision 11ad961)", + version: serverVersion{5, 7, 26}, + flavor: flavorPercona, + }, + { + versionString: "mysqld Ver 10.0.38-MariaDB for Linux on x86_64 (MariaDB Server)", + version: serverVersion{10, 0, 38}, + flavor: flavorMariaDB, + }, + { + versionString: "mysqld Ver 10.1.40-MariaDB for Linux on x86_64 (MariaDB Server)", + version: serverVersion{10, 1, 40}, + flavor: flavorMariaDB, + }, + { + versionString: "mysqld Ver 10.2.25-MariaDB for Linux on x86_64 (MariaDB Server)", + version: serverVersion{10, 2, 25}, + flavor: flavorMariaDB, + }, + { + versionString: "mysqld Ver 10.3.16-MariaDB for Linux on x86_64 (MariaDB Server)", + version: serverVersion{10, 3, 16}, + flavor: flavorMariaDB, + }, + { + versionString: "mysqld Ver 10.4.6-MariaDB for Linux on x86_64 (MariaDB Server)", + version: serverVersion{10, 4, 6}, + flavor: flavorMariaDB, + }, + { + versionString: "mysqld Ver 5.6.42 for linux-glibc2.12 on x86_64 (MySQL Community Server (GPL))", + version: serverVersion{5, 6, 42}, + flavor: flavorMySQL, + }, + { + versionString: "mysqld Ver 5.6.44-86.0 for Linux on x86_64 (Percona Server (GPL), Release 86.0, Revision eba1b3f)", + version: serverVersion{5, 6, 44}, + flavor: flavorPercona, + }, + { + versionString: "mysqld Ver 8.0.15-6 for Linux on x86_64 (Percona Server (GPL), Release 6, Revision 63abd08)", + version: serverVersion{8, 0, 15}, + flavor: flavorPercona, + }, + } + + for _, testcase := range testcases { + f, v, err := parseVersionString(testcase.versionString) + if v != testcase.version || f != testcase.flavor || err != nil { + t.Errorf("parseVersionString failed for: %#v, Got: %#v, %#v Expected: %#v, %#v", testcase.versionString, v, f, testcase.version, testcase.flavor) + } + } + +} + +func TestAssumeVersionString(t *testing.T) { + + // In these cases, the versionstring is nonsensical or unspecified. + // MYSQL_FLAVOR is used instead. + + var testcases = []testcase{ + { + versionString: "MySQL80", + version: serverVersion{8, 0, 11}, + flavor: flavorMySQL, + }, + { + versionString: "MySQL56", + version: serverVersion{5, 7, 10}, // Yes, this has to lie! + flavor: flavorMySQL, // There was no MySQL57 option + }, + { + versionString: "MariaDB", + version: serverVersion{10, 0, 10}, + flavor: flavorMariaDB, + }, + { + versionString: "MariaDB103", + version: serverVersion{10, 3, 7}, + flavor: flavorMariaDB, + }, + } + + for _, testcase := range testcases { + os.Setenv("MYSQL_FLAVOR", testcase.versionString) + f, v, err := getVersionFromEnv() + if v != testcase.version || f != testcase.flavor || err != nil { + t.Errorf("getVersionFromEnv() failed for: %#v, Got: %#v, %#v Expected: %#v, %#v", testcase.versionString, v, f, testcase.version, testcase.flavor) + } + } + +} diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index 33b5852cbd8..be2eb3070fe 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -104,11 +104,7 @@ func Init() (*Env, error) { } te.Dbcfgs = dbconfigs.NewTestDBConfigs(te.cluster.MySQLConnParams(), te.cluster.MySQLAppDebugConnParams(), te.cluster.DbName()) - var err error - te.Mysqld, err = mysqlctl.NewMysqld(te.Dbcfgs) - if err != nil { - return nil, err - } + te.Mysqld = mysqlctl.NewMysqld(te.Dbcfgs) te.SchemaEngine = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) te.SchemaEngine.InitDBConfig(te.Dbcfgs) From 1ecd822beb2cbc37a94981a1ad9fa68d0db7797b Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 6 Aug 2019 10:35:31 -0700 Subject: [PATCH 15/18] update copyright Signed-off-by: Morgan Tocker --- go/vt/mysqlctl/mysqld_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/vt/mysqlctl/mysqld_test.go b/go/vt/mysqlctl/mysqld_test.go index 12539b878e2..f3e45b87e18 100644 --- a/go/vt/mysqlctl/mysqld_test.go +++ b/go/vt/mysqlctl/mysqld_test.go @@ -1,5 +1,5 @@ /* -Copyright 2017 Google Inc. +Copyright 2019 The Vitess Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ package mysqlctl import ( "os" "testing" - // "vitess.io/vitess/go/vt/env" ) type testcase struct { From d4ca143274b635841caef5b91afa1461c7673bbb Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 6 Aug 2019 16:04:15 -0600 Subject: [PATCH 16/18] Remove unset MYSQL_FLAVOR I thought about this: it's a bad idea. It means that if there was a test that created 2 mysqld's, the usage would not be repeatable/idempotent. Signed-off-by: Morgan Tocker --- go/vt/mysqlctl/mysqld.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 2a9d5afd347..d1cb6ce9cea 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -124,9 +124,6 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { } - // Unset ENV to make sure there is no split brain between legacy - // MySQL flavor and capabilities - os.Unsetenv("MYSQL_FLAVOR") log.Infof("Using flavor: %v, version: %v", f, v) result.capabilities = NewCapabilitySet(f, v) return result From e4b487bfe35d28488654682a7692f7f3353e3a33 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 6 Aug 2019 16:11:34 -0600 Subject: [PATCH 17/18] Make sure error is caught. Signed-off-by: Morgan Tocker --- go/vt/mysqlctl/mysqld.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index d1cb6ce9cea..7ccf62bb757 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -112,11 +112,11 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { result.appPool = dbconnpool.NewConnectionPool("AppConnPool", *appPoolSize, *appIdleTimeout, *poolDynamicHostnameResolution) result.appPool.Open(dbcfgs.AppWithDB(), appMysqlStats) - version, _ := getVersionString() + version, getErr := getVersionString() f, v, err := parseVersionString(version) // Fallback if required - if err != nil { + if getErr != nil || err != nil { f, v, err = getVersionFromEnv() if err != nil { panic("Could not detect version from mysqld --version or MYSQL_FLAVOR") From bc08f6af3b93d17817761d1aedeffdc340654308 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 7 Aug 2019 14:15:54 -0600 Subject: [PATCH 18/18] Address PR Feedback Signed-off-by: Morgan Tocker --- go/vt/mysqlctl/mysqld.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 7ccf62bb757..db29d6a0dd6 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -129,25 +129,29 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { return result } +/* +getVersionFromEnv returns the flavor and an assumed version based on the legacy +MYSQL_FLAVOR environment variable. + +The assumed version may not be accurate since the legacy variable only specifies +broad families of compatible versions. However, the differences between those +versions should only matter if Vitess is managing the lifecycle of mysqld, in which +case we should have a local copy of the mysqld binary from which we can fetch +the accurate version instead of falling back to this function (see getVersionString). +*/ func getVersionFromEnv() (flavor mysqlFlavor, ver serverVersion, err error) { - env := os.Getenv("MYSQL_FLAVOR") - - if env == "MariaDB" { + switch env { + case "MariaDB": return flavorMariaDB, serverVersion{10, 0, 10}, nil - } - if env == "MariaDB103" { + case "MariaDB103": return flavorMariaDB, serverVersion{10, 3, 7}, nil - } - if env == "MySQL80" { + case "MySQL80": return flavorMySQL, serverVersion{8, 0, 11}, nil - } - if env == "MySQL56" { + case "MySQL56": return flavorMySQL, serverVersion{5, 7, 10}, nil } - return flavor, ver, fmt.Errorf("Could not determine version from MYSQL_FLAVOR: %s", env) - } func getVersionString() (string, error) { @@ -168,7 +172,6 @@ func getVersionString() (string, error) { // parse the output of mysqld --version into a flavor and version func parseVersionString(version string) (flavor mysqlFlavor, ver serverVersion, err error) { - if strings.Contains(version, "Percona") { flavor = flavorPercona } else if strings.Contains(version, "MariaDB") {