From 609f1d916991afaaf6e76a7bd0c85bdd937e92a6 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Fri, 10 Jan 2020 11:47:28 +0530 Subject: [PATCH 01/50] initial commit for backup_only Signed-off-by: Ajeet jain --- .../endtoend/backup_only/backup_only_test.go | 308 ++++++++++++++++++ go/test/endtoend/backup_only/main_test.go | 141 ++++++++ go/test/endtoend/cluster/cluster_process.go | 26 ++ go/test/endtoend/cluster/vtbackup_process.go | 144 ++++++++ go/test/endtoend/cluster/vttablet_process.go | 3 + 5 files changed, 622 insertions(+) create mode 100644 go/test/endtoend/backup_only/backup_only_test.go create mode 100644 go/test/endtoend/backup_only/main_test.go create mode 100644 go/test/endtoend/cluster/vtbackup_process.go diff --git a/go/test/endtoend/backup_only/backup_only_test.go b/go/test/endtoend/backup_only/backup_only_test.go new file mode 100644 index 00000000000..ed7d325fb88 --- /dev/null +++ b/go/test/endtoend/backup_only/backup_only_test.go @@ -0,0 +1,308 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backup_only + +import ( + "bufio" + "fmt" + "os" + "path" + "strings" + "testing" + "time" + + "vitess.io/vitess/go/test/endtoend/cluster" + + "github.com/stretchr/testify/assert" + + "vitess.io/vitess/go/vt/log" +) + +var ( + vtInsertTest = ` + create table vt_insert_test ( + id bigint auto_increment, + msg varchar(64), + primary key (id) + ) Engine=InnoDB;` +) + +func TestTabletInitialBackup(t *testing.T) { + // Test Initial Backup Flow + // TestTabletInitialBackup will: + // - Create a shard using vtbackup and --initial-backup + // - Create the rest of the cluster restoring from backup + // - Externally Reparenting to a master tablet + // - Insert Some data + // - Verify that the cluster is working + // - Take a Second Backup + // - Bring up a second replica, and restore from the second backup + // - list the backups, remove them + + vtBackup(t, true) + backups := countBackups(t) + assert.Equal(t, 1, backups) + + // Initialize the tablets + initTablets(t, false, false) + + // Restore the Tablets + restore(t, master, "replica", "NOT_SERVING") + err := localCluster.VtctlclientProcess.ExecuteCommand( + "TabletExternallyReparented", master.Alias) + assert.Nil(t, err) + restore(t, replica1, "replica", "SERVING") + + // Run the entire backup test + firstBackupTest(t, "replica") + +} +func TestTabletBackupOnly(t *testing.T) { + // Test Backup Flow + // TestTabletBackupOnly will: + // - Create a shard using regular init & start tablet + // - Run initShardMaster to start replication + // - Insert Some data + // - Verify that the cluster is working + // - Take a Second Backup + // - Bring up a second replica, and restore from the second backup + // - list the backups, remove them + + // Reset the tablet object values in order on init tablet in the next step. + master.VttabletProcess.ServingStatus = "NOT_SERVING" + replica1.VttabletProcess.ServingStatus = "NOT_SERVING" + + initTablets(t, true, true) + firstBackupTest(t, "replica") +} + +func firstBackupTest(t *testing.T, tabletType string) { + // Test First Backup flow. + // + // firstBackupTest will: + // - create a shard with master and replica1 only + // - run InitShardMaster + // - insert some data + // - take a backup + // - insert more data on the master + // - bring up replica2 after the fact, let it restore the backup + // - check all data is right (before+after backup data) + // - list the backup, remove it + + // Store initial backup counts + backupsCount := countBackups(t) + + // insert data on master, wait for slave to get it + _, err := master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true) + assert.Nil(t, err) + // Add a single row with value 'test1' to the master tablet + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test1')", keyspaceName, true) + assert.Nil(t, err) + + // Check that the specified tablet has the expected number of rows + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 1) + + // backup the slave + log.Info("taking backup %s", time.Now()) + vtBackup(t, false) + log.Info("done taking backup %s", time.Now()) + + // check that the backup shows up in the listing + backups := countBackups(t) + assert.Equal(t, backups, backupsCount+1) + + // insert more data on the master + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 2) + + // now bring up the other slave, letting it restore from backup. + err = localCluster.VtctlclientProcess.InitTablet(replica2, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + restore(t, replica2, "replica", "SERVING") + // Replica2 takes time to server. Sleeping for 5 sec. + time.Sleep(5 * time.Second) + //check the new slave has the data + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + + // check that the restored slave has the right local_metadata + result, err := replica2.VttabletProcess.QueryTabletWithDB("select * from local_metadata", "_vt") + assert.Nil(t, err) + assert.Equal(t, replica2.Alias, result.Rows[0][1].ToString(), "Alias") + assert.Equal(t, "ks.0", result.Rows[1][1].ToString(), "ClusterAlias") + assert.Equal(t, cell, result.Rows[2][1].ToString(), "DataCenter") + if tabletType == "replica" { + assert.Equal(t, "neutral", result.Rows[3][1].ToString(), "PromotionRule") + } else { + assert.Equal(t, "must_not", result.Rows[3][1].ToString(), "PromotionRule") + } + + removeBackups(t) + backups = countBackups(t) + assert.Equal(t, 0, backups) + + tearDown(t) +} + +func vtBackup(t *testing.T, initialBackup bool) { + // Take the back using vtbackup executable + extraArgs := []string{"-allow_first_backup", "-db-credentials-file", dbCredentialFile} + log.Info("starting backup tablet %s", time.Now()) + err := localCluster.StartVtbackup(newInitDBFile, initialBackup, keyspaceName, shardName, cell, extraArgs...) + assert.Nil(t, err) +} + +func listBackups(t *testing.T) string { + // Get a list of backup names for the current shard. + localCluster.VtctlProcess = *cluster.VtctlProcessInstance(localCluster.TopoPort, localCluster.Hostname) + backups, err := localCluster.VtctlProcess.ExecuteCommandWithOutput( + "-backup_storage_implementation", "file", + "-file_backup_storage_root", + path.Join(os.Getenv("VTDATAROOT"), "tmp", "backupstorage"), + "ListBackups", shardKsName, + ) + assert.Nil(t, err) + return backups +} + +func countBackups(t *testing.T) int { + // Count the number of backups available in current shard. + backupList := listBackups(t) + backupCount := 0 + // Counts the available backups + scanner := bufio.NewScanner(strings.NewReader(backupList)) + for scanner.Scan() { + if scanner.Text() != "" { + backupCount++ + } + } + return backupCount +} + +func removeBackups(t *testing.T) { + // Remove all the backups from the shard + backupList := listBackups(t) + + scanner := bufio.NewScanner(strings.NewReader(backupList)) + for scanner.Scan() { + if scanner.Text() != "" { + _, err := localCluster.VtctlProcess.ExecuteCommandWithOutput( + "-backup_storage_implementation", "file", + "-file_backup_storage_root", + path.Join(os.Getenv("VTDATAROOT"), "tmp", "backupstorage"), + "RemoveBackup", shardKsName, scanner.Text(), + ) + assert.Nil(t, err) + } + } + +} + +func initTablets(t *testing.T, startTablet bool, initShardMaster bool) { + // Initialize tablets + for _, tablet := range []cluster.Vttablet{*master, *replica1} { + err := localCluster.VtctlclientProcess.InitTablet(&tablet, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + if startTablet { + err = tablet.VttabletProcess.Setup() + assert.Nil(t, err) + } + } + + if initShardMaster { + // choose master and start replication + err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, master.TabletUID) + + if err != nil { + fmt.Println("*********______________****************") + fmt.Println(err) + fmt.Println(master.HTTPPort) + fmt.Println(replica1.HTTPPort) + time.Sleep(5 * time.Minute) + } + assert.Nil(t, err) + } +} + +func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForState string) { + // Erase mysql/tablet dir, then start tablet with restore enabled. + + log.Info("restoring tablet %s", time.Now()) + resetTabletDirectory(t, *tablet) + + err := tablet.VttabletProcess.CreateDB(keyspaceName) + assert.Nil(t, err) + + // Start tablets + tablet.VttabletProcess.ExtraArgs = []string{"-db-credentials-file", dbCredentialFile} + tablet.VttabletProcess.TabletType = tabletType + tablet.VttabletProcess.ServingStatus = waitForState + tablet.VttabletProcess.SupportsBackup = true + err = tablet.VttabletProcess.Setup() + assert.Nil(t, err) +} + +func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet) { + + extraArgs := []string{"-db-credentials-file", dbCredentialFile} + tablet.MysqlctlProcess.ExtraArgs = extraArgs + + // Shutdown Mysql + err := tablet.MysqlctlProcess.Stop() + assert.Nil(t, err) + // Teardown Tablet + err = tablet.VttabletProcess.TearDown() + assert.Nil(t, err) + + // Empty the dir + err = os.RemoveAll(tablet.VttabletProcess.Directory) + assert.Nil(t, err) + + // Init the Mysql + tablet.MysqlctlProcess.InitDBFile = newInitDBFile + err = tablet.MysqlctlProcess.Start() + assert.Nil(t, err) +} + +func tearDown(t *testing.T) { + for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { + //Tear down Tablet + err := tablet.VttabletProcess.TearDown() + assert.Nil(t, err) + err = localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) + assert.Nil(t, err) + + //resetTabletDirectory(t, tablet) + } + + // reset replication + promoteSlaveCommands := "STOP SLAVE; RESET SLAVE ALL; RESET MASTER;" + disableSemiSyncCommands := "SET GLOBAL rpl_semi_sync_master_enabled = false; SET GLOBAL rpl_semi_sync_slave_enabled = false" + for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { + _, err := tablet.VttabletProcess.QueryTablet(promoteSlaveCommands, keyspaceName, true) + assert.Nil(t, err) + _, err = tablet.VttabletProcess.QueryTablet(disableSemiSyncCommands, keyspaceName, true) + assert.Nil(t, err) + + for _, db := range []string{"_vt", "vt_insert_test"} { + _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("drop database if exists %s", db), keyspaceName, true) + assert.Nil(t, err) + } + } + +} diff --git a/go/test/endtoend/backup_only/main_test.go b/go/test/endtoend/backup_only/main_test.go new file mode 100644 index 00000000000..e4dc4b83595 --- /dev/null +++ b/go/test/endtoend/backup_only/main_test.go @@ -0,0 +1,141 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backup_only + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "testing" + + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/sharding/initialsharding" + "vitess.io/vitess/go/vt/log" +) + +var ( + master *cluster.Vttablet + replica1 *cluster.Vttablet + replica2 *cluster.Vttablet + localCluster *cluster.LocalProcessCluster + newInitDBFile string + cell = cluster.DefaultCell + hostname = "localhost" + keyspaceName = "ks" + shardName = "0" + dbPassword = "VtDbaPass" + shardKsName = fmt.Sprintf("%s/%s", keyspaceName, shardName) + dbCredentialFile string + commonTabletArg = []string{ + "-vreplication_healthcheck_topology_refresh", "1s", + "-vreplication_healthcheck_retry_delay", "1s", + "-vreplication_retry_delay", "1s", + "-degraded_threshold", "5s", + "-lock_tables_timeout", "5s", + "-watch_replication_stream", + "-enable_replication_reporter", + "-serving_state_grace_period", "1s"} +) + +func TestMain(m *testing.M) { + flag.Parse() + + exitCode, err := func() (int, error) { + localCluster = cluster.NewCluster(cell, hostname) + defer localCluster.Teardown() + + // Start topo server + err := localCluster.StartTopo() + if err != nil { + return 1, err + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + } + localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace) + + // Create a new init_db.sql file that sets up passwords for all users. + // Then we use a db-credentials-file with the passwords. + dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory) + initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) + sql := string(initDb) + newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") + sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + err = ioutil.WriteFile(newInitDBFile, []byte(sql), 0666) + if err != nil { + return 1, err + } + + extraArgs := []string{"-db-credentials-file", dbCredentialFile} + commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) + + shard := cluster.Shard{ + Name: shardName, + } + + master = localCluster.GetVttabletInstance("replica", 0, "") + replica1 = localCluster.GetVttabletInstance("replica", 0, "") + replica2 = localCluster.GetVttabletInstance("replica", 0, "") + shard.Vttablets = []*cluster.Vttablet{master, replica1, replica2} + + // Start MySql processes + var mysqlProcs []*exec.Cmd + for _, tablet := range shard.Vttablets { + tablet.VttabletProcess = localCluster.GetVtprocessInstanceFromVttablet(tablet, shard.Name, keyspaceName) + tablet.VttabletProcess.DbPassword = dbPassword + tablet.VttabletProcess.ExtraArgs = commonTabletArg + tablet.VttabletProcess.SupportsBackup = true + tablet.VttabletProcess.EnableSemiSync = true + + tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) + tablet.MysqlctlProcess.InitDBFile = newInitDBFile + tablet.MysqlctlProcess.ExtraArgs = extraArgs + if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { + return 1, err + } else { + mysqlProcs = append(mysqlProcs, proc) + } + } + for _, proc := range mysqlProcs { + if err := proc.Wait(); err != nil { + return 1, err + } + } + + // Create database + for _, tablet := range []cluster.Vttablet{*master, *replica1} { + if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil { + return 1, err + } + } + + return m.Run(), nil + }() + + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } else { + os.Exit(exitCode) + } + +} diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index b73a37a18c3..680abbb1fa5 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -64,6 +64,7 @@ type LocalProcessCluster struct { VtctldProcess VtctldProcess VtgateProcess VtgateProcess VtworkerProcess VtworkerProcess + VtbackupProcess VtbackupProcess nextPortForProcess int @@ -482,6 +483,11 @@ func (cluster *LocalProcessCluster) Teardown() { if err := cluster.TopoProcess.TearDown(cluster.Cell, cluster.OriginalVTDATAROOT, cluster.CurrentVTDATAROOT, *keepData); err != nil { log.Errorf("Error in etcd teardown - %s", err.Error()) } + + if err := cluster.VtbackupProcess.TearDown(); err != nil { + log.Errorf("Error in VtBackup teardown - %s", err.Error()) + } + } // StartVtworker starts a vtworker @@ -500,6 +506,26 @@ func (cluster *LocalProcessCluster) StartVtworker(cell string, extraArgs ...stri } +// StartVtbackup starts a vtbackup +func (cluster *LocalProcessCluster) StartVtbackup(newInitDBFile string, initalBackup bool, + keyspace string, shard string, cell string, extraArgs ...string) error { + log.Info("Starting vtbackup") + cluster.VtbackupProcess = *VtbackupProcessInstance( + cluster.GetAndReserveTabletUID(), + cluster.GetAndReservePort(), + newInitDBFile, + keyspace, + shard, + cell, + cluster.Hostname, + cluster.TmpDirectory, + cluster.TopoPort, + initalBackup) + cluster.VtbackupProcess.ExtraArgs = extraArgs + return cluster.VtbackupProcess.Setup() + +} + // GetAndReservePort gives port for required process func (cluster *LocalProcessCluster) GetAndReservePort() int { if cluster.nextPortForProcess == 0 { diff --git a/go/test/endtoend/cluster/vtbackup_process.go b/go/test/endtoend/cluster/vtbackup_process.go new file mode 100644 index 00000000000..b8f4d866545 --- /dev/null +++ b/go/test/endtoend/cluster/vtbackup_process.go @@ -0,0 +1,144 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "os" + "os/exec" + "path" + "strings" + "syscall" + "time" + + "vitess.io/vitess/go/vt/log" +) + +// VtbackupProcess is a generic handle for a running Vtbackup. +// It can be spawned manually +type VtbackupProcess struct { + Name string + Binary string + CommonArg VtctlProcess + LogDir string + MysqlPort int + Directory string + + Cell string + Keyspace string + Shard string + TabletAlias string + Server string + + ExtraArgs []string + initialBackup bool + initDBfile string + dbPassword string + dbName string + + proc *exec.Cmd + exit chan error +} + +// Setup starts vtbackup process with required arguements +func (vtbackup *VtbackupProcess) Setup() (err error) { + + vtbackup.proc = exec.Command( + vtbackup.Binary, + "-topo_implementation", vtbackup.CommonArg.TopoImplementation, + "-topo_global_server_address", vtbackup.CommonArg.TopoGlobalAddress, + "-topo_global_root", vtbackup.CommonArg.TopoGlobalRoot, + "-log_dir", vtbackup.LogDir, + + //initDBfile is required to run vtbackup + "-mysql_port", fmt.Sprintf("%d", vtbackup.MysqlPort), + "-init_db_sql_file", vtbackup.initDBfile, + "-init_keyspace", vtbackup.Keyspace, + "-init_shard", vtbackup.Shard, + + //Backup Arguments are not optional + "-backup_storage_implementation", "file", + "-file_backup_storage_root", + path.Join(os.Getenv("VTDATAROOT"), "tmp", "backupstorage"), + ) + + if vtbackup.initialBackup { + vtbackup.proc.Args = append(vtbackup.proc.Args, "-initial_backup") + } + if vtbackup.ExtraArgs != nil { + vtbackup.proc.Args = append(vtbackup.proc.Args, vtbackup.ExtraArgs...) + } + + vtbackup.proc.Stderr = os.Stderr + vtbackup.proc.Stdout = os.Stdout + + vtbackup.proc.Env = append(vtbackup.proc.Env, os.Environ()...) + log.Infof("%v", strings.Join(vtbackup.proc.Args, " ")) + fmt.Println(vtbackup.proc.Args) + + err = vtbackup.proc.Run() + if err != nil { + return + } + + return nil +} + +// TearDown shutdowns the running vtbackup process +func (vtbackup *VtbackupProcess) TearDown() error { + if vtbackup.proc == nil || vtbackup.exit == nil { + return nil + } + + // Attempt graceful shutdown with SIGTERM first + vtbackup.proc.Process.Signal(syscall.SIGTERM) + + select { + case err := <-vtbackup.exit: + vtbackup.proc = nil + return err + + case <-time.After(10 * time.Second): + vtbackup.proc.Process.Kill() + vtbackup.proc = nil + return <-vtbackup.exit + } +} + +// VtbackupProcessInstance returns a vtbackup handle +// configured with the given Config. +// The process must be manually started by calling Setup() + +func VtbackupProcessInstance(tabletUid int, mysqlPort int, newInitDBFile string, keyspace string, shard string, + cell string, hostname string, tmpDirectory string, topoPort int, initialBackup bool) *VtbackupProcess { + vtctl := VtctlProcessInstance(topoPort, hostname) + vtbackup := &VtbackupProcess{ + Name: "vtbackup", + Binary: "vtbackup", + CommonArg: *vtctl, + LogDir: tmpDirectory, + Directory: os.Getenv("VTDATAROOT"), + TabletAlias: fmt.Sprintf("%s-%010d", cell, tabletUid), + initDBfile: newInitDBFile, + Keyspace: keyspace, + Shard: shard, + Cell: cell, + MysqlPort: mysqlPort, + initialBackup: initialBackup, + } + return vtbackup +} diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index 1b70264b669..f4a98715d12 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -319,6 +319,9 @@ func (vttablet *VttabletProcess) QueryTabletWithDB(query string, dbname string) UnixSocket: path.Join(vttablet.Directory, "mysql.sock"), DbName: dbname, } + if vttablet.DbPassword != "" { + dbParams.Pass = vttablet.DbPassword + } return executeQuery(dbParams, query) } From a36ac69a6bc6bfc5d93aa304afb7f51e5132c4d0 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Fri, 10 Jan 2020 11:56:59 +0530 Subject: [PATCH 02/50] changes in package structure Signed-off-by: Ajeet jain --- .../{backup_only => backup/vtbackup}/backup_only_test.go | 2 +- .../endtoend/{backup_only => backup/vtbackup}/main_test.go | 2 +- go/test/endtoend/cluster/vtbackup_process.go | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) rename go/test/endtoend/{backup_only => backup/vtbackup}/backup_only_test.go (99%) rename go/test/endtoend/{backup_only => backup/vtbackup}/main_test.go (99%) diff --git a/go/test/endtoend/backup_only/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go similarity index 99% rename from go/test/endtoend/backup_only/backup_only_test.go rename to go/test/endtoend/backup/vtbackup/backup_only_test.go index ed7d325fb88..2dee35a7ba6 100644 --- a/go/test/endtoend/backup_only/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backup_only +package vtbackup import ( "bufio" diff --git a/go/test/endtoend/backup_only/main_test.go b/go/test/endtoend/backup/vtbackup/main_test.go similarity index 99% rename from go/test/endtoend/backup_only/main_test.go rename to go/test/endtoend/backup/vtbackup/main_test.go index e4dc4b83595..6a597371057 100644 --- a/go/test/endtoend/backup_only/main_test.go +++ b/go/test/endtoend/backup/vtbackup/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backup_only +package vtbackup import ( "flag" diff --git a/go/test/endtoend/cluster/vtbackup_process.go b/go/test/endtoend/cluster/vtbackup_process.go index b8f4d866545..01f93e67be3 100644 --- a/go/test/endtoend/cluster/vtbackup_process.go +++ b/go/test/endtoend/cluster/vtbackup_process.go @@ -122,8 +122,7 @@ func (vtbackup *VtbackupProcess) TearDown() error { // VtbackupProcessInstance returns a vtbackup handle // configured with the given Config. // The process must be manually started by calling Setup() - -func VtbackupProcessInstance(tabletUid int, mysqlPort int, newInitDBFile string, keyspace string, shard string, +func VtbackupProcessInstance(tabletUID int, mysqlPort int, newInitDBFile string, keyspace string, shard string, cell string, hostname string, tmpDirectory string, topoPort int, initialBackup bool) *VtbackupProcess { vtctl := VtctlProcessInstance(topoPort, hostname) vtbackup := &VtbackupProcess{ @@ -132,7 +131,7 @@ func VtbackupProcessInstance(tabletUid int, mysqlPort int, newInitDBFile string, CommonArg: *vtctl, LogDir: tmpDirectory, Directory: os.Getenv("VTDATAROOT"), - TabletAlias: fmt.Sprintf("%s-%010d", cell, tabletUid), + TabletAlias: fmt.Sprintf("%s-%010d", cell, tabletUID), initDBfile: newInitDBFile, Keyspace: keyspace, Shard: shard, From b64271ba6f090a7a1c9d6be2db78f1f7ec0b89fc Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Fri, 10 Jan 2020 13:41:05 +0530 Subject: [PATCH 03/50] updating package name and fixing a test Signed-off-by: Ajeet jain --- .../backup/vtbackup/backup_only_test.go | 57 ++++++++++--------- .../backup/{ => vtctlbackup}/backup_test.go | 2 +- .../backup/{ => vtctlbackup}/main_test.go | 2 +- test/config.json | 20 +++---- 4 files changed, 43 insertions(+), 38 deletions(-) rename go/test/endtoend/backup/{ => vtctlbackup}/backup_test.go (99%) rename go/test/endtoend/backup/{ => vtctlbackup}/main_test.go (99%) diff --git a/go/test/endtoend/backup/vtbackup/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go index 2dee35a7ba6..b57907cbcb9 100644 --- a/go/test/endtoend/backup/vtbackup/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -70,6 +70,7 @@ func TestTabletInitialBackup(t *testing.T) { // Run the entire backup test firstBackupTest(t, "replica") + tearDown(t, true) } func TestTabletBackupOnly(t *testing.T) { // Test Backup Flow @@ -88,6 +89,8 @@ func TestTabletBackupOnly(t *testing.T) { initTablets(t, true, true) firstBackupTest(t, "replica") + + tearDown(t, false) } func firstBackupTest(t *testing.T, tabletType string) { @@ -155,7 +158,6 @@ func firstBackupTest(t *testing.T, tabletType string) { backups = countBackups(t) assert.Equal(t, 0, backups) - tearDown(t) } func vtBackup(t *testing.T, initialBackup bool) { @@ -243,7 +245,7 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS // Erase mysql/tablet dir, then start tablet with restore enabled. log.Info("restoring tablet %s", time.Now()) - resetTabletDirectory(t, *tablet) + resetTabletDirectory(t, *tablet, true) err := tablet.VttabletProcess.CreateDB(keyspaceName) assert.Nil(t, err) @@ -257,7 +259,7 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS assert.Nil(t, err) } -func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet) { +func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet, initMysql bool) { extraArgs := []string{"-db-credentials-file", dbCredentialFile} tablet.MysqlctlProcess.ExtraArgs = extraArgs @@ -273,36 +275,39 @@ func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet) { err = os.RemoveAll(tablet.VttabletProcess.Directory) assert.Nil(t, err) - // Init the Mysql - tablet.MysqlctlProcess.InitDBFile = newInitDBFile - err = tablet.MysqlctlProcess.Start() - assert.Nil(t, err) + if initMysql { + // Init the Mysql + tablet.MysqlctlProcess.InitDBFile = newInitDBFile + err = tablet.MysqlctlProcess.Start() + assert.Nil(t, err) + } + } -func tearDown(t *testing.T) { +func tearDown(t *testing.T, initMysql bool) { for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { //Tear down Tablet - err := tablet.VttabletProcess.TearDown() - assert.Nil(t, err) - err = localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) + //err := tablet.VttabletProcess.TearDown() + //assert.Nil(t, err) + err := localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) assert.Nil(t, err) - //resetTabletDirectory(t, tablet) + resetTabletDirectory(t, tablet, initMysql) } - // reset replication - promoteSlaveCommands := "STOP SLAVE; RESET SLAVE ALL; RESET MASTER;" - disableSemiSyncCommands := "SET GLOBAL rpl_semi_sync_master_enabled = false; SET GLOBAL rpl_semi_sync_slave_enabled = false" - for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { - _, err := tablet.VttabletProcess.QueryTablet(promoteSlaveCommands, keyspaceName, true) - assert.Nil(t, err) - _, err = tablet.VttabletProcess.QueryTablet(disableSemiSyncCommands, keyspaceName, true) - assert.Nil(t, err) - - for _, db := range []string{"_vt", "vt_insert_test"} { - _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("drop database if exists %s", db), keyspaceName, true) - assert.Nil(t, err) - } - } + //// reset replication + //promoteSlaveCommands := "STOP SLAVE; RESET SLAVE ALL; RESET MASTER;" + //disableSemiSyncCommands := "SET GLOBAL rpl_semi_sync_master_enabled = false; SET GLOBAL rpl_semi_sync_slave_enabled = false" + //for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { + // _, err := tablet.VttabletProcess.QueryTablet(promoteSlaveCommands, keyspaceName, true) + // assert.Nil(t, err) + // _, err = tablet.VttabletProcess.QueryTablet(disableSemiSyncCommands, keyspaceName, true) + // assert.Nil(t, err) + // + // for _, db := range []string{"_vt", "vt_insert_test"} { + // _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("drop database if exists %s", db), keyspaceName, true) + // assert.Nil(t, err) + // } + //} } diff --git a/go/test/endtoend/backup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go similarity index 99% rename from go/test/endtoend/backup/backup_test.go rename to go/test/endtoend/backup/vtctlbackup/backup_test.go index 785209ca530..0ccc92809cb 100644 --- a/go/test/endtoend/backup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backup +package vtctlbackup import ( "bufio" diff --git a/go/test/endtoend/backup/main_test.go b/go/test/endtoend/backup/vtctlbackup/main_test.go similarity index 99% rename from go/test/endtoend/backup/main_test.go rename to go/test/endtoend/backup/vtctlbackup/main_test.go index 5c926dd4535..18ed385c94e 100644 --- a/go/test/endtoend/backup/main_test.go +++ b/go/test/endtoend/backup/vtctlbackup/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backup +package vtctlbackup import ( "flag" diff --git a/test/config.json b/test/config.json index e6904a4004e..da04a180222 100644 --- a/test/config.json +++ b/test/config.json @@ -1,14 +1,5 @@ { "Tests": { - "backup_only": { - "File": "backup_only.py", - "Args": [], - "Command": [], - "Manual": false, - "Shard": 2, - "RetryMax": 0, - "Tags": [] - }, "backup_mysqlctld": { "File": "backup_mysqlctld.py", "Args": [], @@ -236,7 +227,16 @@ }, "backup": { "File": "backup.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backup"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtctlbackup"], + "Command": [], + "Manual": false, + "Shard": 11, + "RetryMax": 0, + "Tags": [] + }, + "backup_only": { + "File": "backup_only.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtbackup"], "Command": [], "Manual": false, "Shard": 11, From a8273a841d904ec36454b23292da8e7f705bdd2d Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Mon, 13 Jan 2020 10:02:14 +0530 Subject: [PATCH 04/50] removed unrequired teardown code Signed-off-by: Ajeet jain --- go/test/endtoend/cluster/cluster_process.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index 680abbb1fa5..f3e75e845ff 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -484,10 +484,6 @@ func (cluster *LocalProcessCluster) Teardown() { log.Errorf("Error in etcd teardown - %s", err.Error()) } - if err := cluster.VtbackupProcess.TearDown(); err != nil { - log.Errorf("Error in VtBackup teardown - %s", err.Error()) - } - } // StartVtworker starts a vtworker From 975645f511933c05f7b08a86f1bb703c04946fde Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Mon, 13 Jan 2020 10:13:21 +0530 Subject: [PATCH 05/50] Removed debug code Signed-off-by: Ajeet jain --- go/test/endtoend/backup/vtbackup/backup_only_test.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/go/test/endtoend/backup/vtbackup/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go index b57907cbcb9..0f987f16d21 100644 --- a/go/test/endtoend/backup/vtbackup/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -18,7 +18,6 @@ package vtbackup import ( "bufio" - "fmt" "os" "path" "strings" @@ -137,7 +136,7 @@ func firstBackupTest(t *testing.T, tabletType string) { err = localCluster.VtctlclientProcess.InitTablet(replica2, cell, keyspaceName, hostname, shardName) assert.Nil(t, err) restore(t, replica2, "replica", "SERVING") - // Replica2 takes time to server. Sleeping for 5 sec. + // Replica2 takes time to serve. Sleeping for 5 sec. time.Sleep(5 * time.Second) //check the new slave has the data cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) @@ -229,14 +228,6 @@ func initTablets(t *testing.T, startTablet bool, initShardMaster bool) { if initShardMaster { // choose master and start replication err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, master.TabletUID) - - if err != nil { - fmt.Println("*********______________****************") - fmt.Println(err) - fmt.Println(master.HTTPPort) - fmt.Println(replica1.HTTPPort) - time.Sleep(5 * time.Minute) - } assert.Nil(t, err) } } From 1563e63ed62ad0471c0e2fee6f38d3ece3a35e20 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Mon, 20 Jan 2020 15:43:51 +0530 Subject: [PATCH 06/50] inital commit for xtrabackup Signed-off-by: Ajeet jain --- .../backup/vtctlbackup/backup_test.go | 432 +----------- .../backup/vtctlbackup/backup_utils.go | 613 ++++++++++++++++++ .../endtoend/backup/vtctlbackup/main_test.go | 151 ----- .../backup/xtrabackup/xtrabackup_test.go | 39 ++ 4 files changed, 662 insertions(+), 573 deletions(-) create mode 100644 go/test/endtoend/backup/vtctlbackup/backup_utils.go delete mode 100644 go/test/endtoend/backup/vtctlbackup/main_test.go create mode 100644 go/test/endtoend/backup/xtrabackup/xtrabackup_test.go diff --git a/go/test/endtoend/backup/vtctlbackup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go index 0ccc92809cb..b98435a30c2 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -16,430 +16,18 @@ limitations under the License. package vtctlbackup -import ( - "bufio" - "encoding/json" - "os" - "os/exec" - "path" - "strings" - "syscall" - "testing" - "time" +import "testing" - "vitess.io/vitess/go/vt/proto/topodata" - - "github.com/stretchr/testify/assert" - "vitess.io/vitess/go/test/endtoend/cluster" -) - -var ( - vtInsertTest = ` - create table vt_insert_test ( - id bigint auto_increment, - msg varchar(64), - primary key (id) - ) Engine=InnoDB` -) - -type restoreMethod func(t *testing.T, tablet *cluster.Vttablet) - -func TestReplicaBackup(t *testing.T) { - testBackup(t, "replica") -} - -func TestRdonlyBackup(t *testing.T) { - testBackup(t, "rdonly") -} - -//- create a shard with master and replica1 only -//- run InitShardMaster -//- insert some data -//- take a backup on master -//- insert more data on the master -//- bring up tablet_replica2 after the fact, let it restore the backup -//- check all data is right (before+after backup data) -//- list the backup, remove it -func TestMasterBackup(t *testing.T) { - verifyInitialReplication(t) - - output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("Backup", master.Alias) - assert.NotNil(t, err) - assert.Contains(t, output, "type MASTER cannot take backup. if you really need to do this, rerun the backup command with -allow_master") - - backups := listBackups(t) - assert.Equal(t, len(backups), 0) - - err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", "-allow_master=true", master.Alias) - assert.Nil(t, err) - - backups = listBackups(t) - assert.Equal(t, len(backups), 1) - assert.Contains(t, backups[0], master.Alias) - - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - - restoreWaitForBackup(t, "replica") - err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) - assert.Nil(t, err) - - cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) - cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell) - verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups) - - replica2.VttabletProcess.TearDown() - master.VttabletProcess.QueryTablet("DROP TABLE vt_insert_test", keyspaceName, true) -} - -// Test a master and replica from the same backup. -// -// Check that a replica and master both restored from the same backup -// can replicate successfully. -func TestMasterReplicaSameBackup(t *testing.T) { - // insert data on master, wait for replica to get it - verifyInitialReplication(t) - - // backup the replica - err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - // insert more data on the master - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - - // now bring up the other replica, letting it restore from backup. - restoreWaitForBackup(t, "replica") - err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) - assert.Nil(t, err) - - // check the new replica has the data - cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) - - // Promote replica2 to master - err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", - "-keyspace_shard", shardKsName, - "-new_master", replica2.Alias) - assert.Nil(t, err) - - // insert more data on replica2 (current master) - _, err = replica2.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) - assert.Nil(t, err) - - // Force replica1 to restore from backup. - verifyRestoreTablet(t, replica1, "SERVING") - - // wait for replica1 to catch up. - cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 3) - - // This is to test that replicationPosition is processed correctly - // while doing backup/restore after a reparent. - // It is written into the MANIFEST and read back from the MANIFEST. - // - // Take another backup on the replica. - err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - // Insert more data on replica2 (current master). - _, err = replica2.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test4')", keyspaceName, true) - assert.Nil(t, err) - - // Force replica1 to restore from backup. - verifyRestoreTablet(t, replica1, "SERVING") - - cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 4) - replica2.VttabletProcess.TearDown() - restartMasterReplica(t) -} - -func TestRestoreOldMasterByRestart(t *testing.T) { - testRestoreOldMaster(t, restoreUsingRestart) -} - -func TestRestoreOldMasterInPlace(t *testing.T) { - testRestoreOldMaster(t, restoreInPlace) -} - -//Test that a former master replicates correctly after being restored. -// -//- Take a backup. -//- Reparent from old master to new master. -//- Force old master to restore from a previous backup using restore_method. -// -//Args: -//restore_method: function accepting one parameter of type tablet.Tablet, -//this function is called to force a restore on the provided tablet -// -func testRestoreOldMaster(t *testing.T, method restoreMethod) { - // insert data on master, wait for replica to get it - verifyInitialReplication(t) - - // backup the replica - err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - // insert more data on the master - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - - // reparent to replica1 - err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", - "-keyspace_shard", shardKsName, - "-new_master", replica1.Alias) - assert.Nil(t, err) - - // insert more data to new master - _, err = replica1.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) - assert.Nil(t, err) - - // force the old master to restore at the latest backup. - method(t, master) - - // wait for it to catch up. - cluster.VerifyRowsInTablet(t, master, keyspaceName, 3) - - // teardown - restartMasterReplica(t) -} - -func restoreUsingRestart(t *testing.T, tablet *cluster.Vttablet) { - tablet.VttabletProcess.TearDown() - verifyRestoreTablet(t, tablet, "SERVING") -} - -func restoreInPlace(t *testing.T, tablet *cluster.Vttablet) { - err := localCluster.VtctlclientProcess.ExecuteCommand("RestoreFromBackup", tablet.Alias) - assert.Nil(t, err) -} - -func restartMasterReplica(t *testing.T) { - // Stop all master, replica tablet and mysql instance - stopAllTablets() - - // remove all backups - backups := listBackups(t) - for _, backup := range backups { - localCluster.VtctlclientProcess.ExecuteCommand("RemoveBackup", shardKsName, backup) +// TestBackupMain - main tests backup using vtctl commands +func TestBackupMain(t *testing.T) { + code, err := LaunchCluster(false) + if err != nil { + t.Errorf("setup failed with status code %d", code) } - // start all tablet and mysql instances - var mysqlProcs []*exec.Cmd - for _, tablet := range []*cluster.Vttablet{master, replica1} { - proc, _ := tablet.MysqlctlProcess.StartProcess() - mysqlProcs = append(mysqlProcs, proc) - err := localCluster.VtctlclientProcess.InitTablet(tablet, cell, keyspaceName, hostname, shardName) - assert.Nil(t, err) - tablet.VttabletProcess.CreateDB(keyspaceName) - tablet.VttabletProcess.Setup() - } - for _, proc := range mysqlProcs { - proc.Wait() - } - err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, master.TabletUID) - assert.Nil(t, err) -} - -func stopAllTablets() { - var mysqlProcs []*exec.Cmd - for _, tablet := range []*cluster.Vttablet{master, replica1, replica2} { - tablet.VttabletProcess.TearDown() - proc, _ := tablet.MysqlctlProcess.StopProcess() - mysqlProcs = append(mysqlProcs, proc) - localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) - } - for _, proc := range mysqlProcs { - proc.Wait() - } - for _, tablet := range []*cluster.Vttablet{master, replica1} { - os.RemoveAll(tablet.VttabletProcess.Directory) - } -} - -func TestTerminatedRestore(t *testing.T) { - // insert data on master, wait for replica to get it - verifyInitialReplication(t) - - // backup the replica - err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - // insert more data on the master - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - - // reparent to replica1 - err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", - "-keyspace_shard", shardKsName, - "-new_master", replica1.Alias) - assert.Nil(t, err) - - // insert more data to new master - _, err = replica1.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) - assert.Nil(t, err) - - terminateRestore(t) - - err = localCluster.VtctlclientProcess.ExecuteCommand("RestoreFromBackup", master.Alias) - assert.Nil(t, err) - - output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", master.Alias) - assert.Nil(t, err) - - var tabletPB topodata.Tablet - err = json.Unmarshal([]byte(output), &tabletPB) - assert.Nil(t, err) - assert.Equal(t, tabletPB.Type, topodata.TabletType_REPLICA) - - _, err = os.Stat(path.Join(master.VttabletProcess.Directory, "restore_in_progress")) - assert.True(t, os.IsNotExist(err)) - - cluster.VerifyRowsInTablet(t, master, keyspaceName, 3) - stopAllTablets() -} - -//test_backup will: -//- create a shard with master and replica1 only -//- run InitShardMaster -//- bring up tablet_replica2 concurrently, telling it to wait for a backup -//- insert some data -//- take a backup -//- insert more data on the master -//- wait for tablet_replica2 to become SERVING -//- check all data is right (before+after backup data) -//- list the backup, remove it -// -//Args: -//tablet_type: 'replica' or 'rdonly'. -// -// -func testBackup(t *testing.T, tabletType string) { - restoreWaitForBackup(t, tabletType) - verifyInitialReplication(t) - - err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - backups := listBackups(t) - assert.Equal(t, len(backups), 1) - - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - - err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) - assert.Nil(t, err) - cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) - - cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell) - verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups) + // Run all the backup tests + TestBackup(t) - replica2.VttabletProcess.TearDown() - localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", replica2.Alias) - master.VttabletProcess.QueryTablet("DROP TABLE vt_insert_test", keyspaceName, true) - -} - -// This will create schema in master, insert some data to master and verify the same data in replica -func verifyInitialReplication(t *testing.T) { - _, err := master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true) - assert.Nil(t, err) - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test1')", keyspaceName, true) - assert.Nil(t, err) - cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 1) -} - -// Bring up another replica concurrently, telling it to wait until a backup -// is available instead of starting up empty. -// -// Override the backup engine implementation to a non-existent one for restore. -// This setting should only matter for taking new backups. We should be able -// to restore a previous backup successfully regardless of this setting. -func restoreWaitForBackup(t *testing.T, tabletType string) { - replica2.Type = tabletType - resetTabletDir(t, replica2) - replicaTabletArgs := commonTabletArg - replicaTabletArgs = append(replicaTabletArgs, "-backup_engine_implementation", "fake_implementation") - replicaTabletArgs = append(replicaTabletArgs, "-wait_for_backup_interval", "1s") - replicaTabletArgs = append(replicaTabletArgs, "-init_tablet_type", tabletType) - replica2.VttabletProcess.ExtraArgs = replicaTabletArgs - replica2.VttabletProcess.ServingStatus = "" - err := replica2.VttabletProcess.Setup() - assert.Nil(t, err) -} - -func resetTabletDir(t *testing.T, tablet *cluster.Vttablet) { - err := cluster.ResetTabletDirectory(*tablet) - assert.Nil(t, err) -} - -func listBackups(t *testing.T) []string { - output, err := localCluster.ListBackups(shardKsName) - assert.Nil(t, err) - return output -} - -func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []string) { - // Remove the backup - for _, backup := range backups { - err := localCluster.VtctlclientProcess.ExecuteCommand("RemoveBackup", shardKsName, backup) - assert.Nil(t, err) - } - - // Now, there should not be no backup - backups = listBackups(t) - assert.Equal(t, len(backups), 0) -} - -func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string) { - tablet.VttabletProcess.TearDown() - resetTabletDir(t, tablet) - tablet.VttabletProcess.ServingStatus = "" - err := tablet.VttabletProcess.Setup() - assert.Nil(t, err) - if status != "" { - err = tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{status}, 25*time.Second) - assert.Nil(t, err) - } - - if tablet.Type == "replica" { - verifyReplicationStatus(t, tablet, "ON") - } else if tablet.Type == "rdonly" { - verifyReplicationStatus(t, tablet, "OFF") - } -} - -func verifyReplicationStatus(t *testing.T, vttablet *cluster.Vttablet, expectedStatus string) { - status, err := vttablet.VttabletProcess.GetDBVar("rpl_semi_sync_slave_enabled", keyspaceName) - assert.Nil(t, err) - assert.Equal(t, status, expectedStatus) - status, err = vttablet.VttabletProcess.GetDBStatus("rpl_semi_sync_slave_status", keyspaceName) - assert.Nil(t, err) - assert.Equal(t, status, expectedStatus) -} - -func terminateRestore(t *testing.T) { - stopRestoreMsg := "Copying file 10" - args := append([]string{"-server", localCluster.VtctlclientProcess.Server, "-alsologtostderr"}, "RestoreFromBackup", master.Alias) - tmpProcess := exec.Command( - "vtctlclient", - args..., - ) - - reader, _ := tmpProcess.StderrPipe() - err := tmpProcess.Start() - assert.Nil(t, err) - found := false - - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - text := scanner.Text() - if strings.Contains(text, stopRestoreMsg) { - if _, err := os.Stat(path.Join(master.VttabletProcess.Directory, "restore_in_progress")); os.IsNotExist(err) { - assert.Fail(t, "restore in progress file missing") - } - tmpProcess.Process.Signal(syscall.SIGTERM) - found = true - return - } - } - assert.True(t, found, "Restore message not found") + // Teardown the cluster + TearDownCluster() } diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go new file mode 100644 index 00000000000..2164e635b2c --- /dev/null +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -0,0 +1,613 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctlbackup + +import ( + "bufio" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + "syscall" + "testing" + "time" + + "vitess.io/vitess/go/test/endtoend/sharding/initialsharding" + + "vitess.io/vitess/go/vt/proto/topodata" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + master *cluster.Vttablet + replica1 *cluster.Vttablet + replica2 *cluster.Vttablet + localCluster *cluster.LocalProcessCluster + newInitDBFile string + cell = cluster.DefaultCell + hostname = "localhost" + keyspaceName = "ks" + dbPassword = "VtDbaPass" + shardKsName = fmt.Sprintf("%s/%s", keyspaceName, shardName) + dbCredentialFile string + shardName = "0" + commonTabletArg = []string{ + "-vreplication_healthcheck_topology_refresh", "1s", + "-vreplication_healthcheck_retry_delay", "1s", + "-vreplication_retry_delay", "1s", + "-degraded_threshold", "5s", + "-lock_tables_timeout", "5s", + "-watch_replication_stream", + "-enable_replication_reporter", + "-serving_state_grace_period", "1s"} + xtrabackupArgs = []string{ + "-backup_engine_implementation", + "xtrabackup", + "-xtrabackup_stream_mode", "tar", + "-xtrabackup_user=vt_dba", + "-xtrabackup_stripes=0", + "-xtrabackup_backup_flags", + fmt.Sprintf("--password=%s", dbPassword)} + + vtInsertTest = ` + create table vt_insert_test ( + id bigint auto_increment, + msg varchar(64), + primary key (id) + ) Engine=InnoDB` +) + +func LaunchCluster(xtrabackup bool) (int, error) { + localCluster = cluster.NewCluster(cell, hostname) + + // Start topo server + err := localCluster.StartTopo() + if err != nil { + return 1, err + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + } + localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace) + + dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory) + initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) + sql := string(initDb) + newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") + sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + err = ioutil.WriteFile(newInitDBFile, []byte(sql), 0666) + if err != nil { + return 1, err + } + + extraArgs := []string{"-db-credentials-file", dbCredentialFile} + commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) + + if xtrabackup { + commonTabletArg = append(commonTabletArg, xtrabackupArgs...) + } + + shard := cluster.Shard{ + Name: shardName, + } + + var mysqlProcs []*exec.Cmd + for i := 0; i < 3; i++ { + tabletType := "replica" + if i == 0 { + tabletType = "master" + } + tablet := localCluster.GetVttabletInstance(tabletType, 0, cell) + tablet.VttabletProcess = localCluster.GetVtprocessInstanceFromVttablet(tablet, shard.Name, keyspaceName) + tablet.VttabletProcess.DbPassword = dbPassword + tablet.VttabletProcess.ExtraArgs = commonTabletArg + tablet.VttabletProcess.SupportsBackup = true + tablet.VttabletProcess.EnableSemiSync = true + + tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) + tablet.MysqlctlProcess.InitDBFile = newInitDBFile + tablet.MysqlctlProcess.ExtraArgs = extraArgs + if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { + return 1, err + } else { + mysqlProcs = append(mysqlProcs, proc) + } + shard.Vttablets = append(shard.Vttablets, tablet) + } + for _, proc := range mysqlProcs { + if err := proc.Wait(); err != nil { + return 1, err + } + } + master = shard.Vttablets[0] + replica1 = shard.Vttablets[1] + replica2 = shard.Vttablets[2] + + if err := localCluster.VtctlclientProcess.InitTablet(master, cell, keyspaceName, hostname, shard.Name); err != nil { + return 1, err + } + if err := localCluster.VtctlclientProcess.InitTablet(replica1, cell, keyspaceName, hostname, shard.Name); err != nil { + return 1, err + } + + for _, tablet := range []cluster.Vttablet{*master, *replica1} { + if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil { + return 1, err + } + if err := tablet.VttabletProcess.Setup(); err != nil { + return 1, err + } + } + + if err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shard.Name, cell, master.TabletUID); err != nil { + return 1, err + } + return 0, nil +} + +func TearDownCluster() { + localCluster.Teardown() +} + +func TestBackup(t *testing.T) { + // Run all the backup tests + t.Run("TestReplicaBackup", func(t *testing.T) { + vtctlBackup(t, "replica") + }) + + t.Run("TestRdonlyBackup", func(t *testing.T) { + vtctlBackup(t, "rdonly") + }) + + t.Run("TestMasterBackup", func(t *testing.T) { + masterBackup(t) + }) + + t.Run("TestMasterReplicaSameBackup", func(t *testing.T) { + masterReplicaSameBackup(t) + }) + + t.Run("TestRestoreOldMasterByRestart", func(t *testing.T) { + restoreOldMasterByRestart(t) + }) + + t.Run("TestRestoreOldMasterInPlace", func(t *testing.T) { + restoreOldMasterInPlace(t) + }) + + t.Run("TestTerminatedRestore", func(t *testing.T) { + terminatedRestore(t) + }) + +} + +type restoreMethod func(t *testing.T, tablet *cluster.Vttablet) + +//- create a shard with master and replica1 only +//- run InitShardMaster +//- insert some data +//- take a backup on master +//- insert more data on the master +//- bring up tablet_replica2 after the fact, let it restore the backup +//- check all data is right (before+after backup data) +//- list the backup, remove it +func masterBackup(t *testing.T) { + verifyInitialReplication(t) + + output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("Backup", master.Alias) + assert.NotNil(t, err) + assert.Contains(t, output, "type MASTER cannot take backup. if you really need to do this, rerun the backup command with -allow_master") + + backups := listBackups(t) + assert.Equal(t, len(backups), 0) + + err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", "-allow_master=true", master.Alias) + assert.Nil(t, err) + + backups = listBackups(t) + assert.Equal(t, len(backups), 1) + assert.Contains(t, backups[0], master.Alias) + + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + + restoreWaitForBackup(t, "replica") + err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) + assert.Nil(t, err) + + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell) + verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups) + + err = replica2.VttabletProcess.TearDown() + assert.Nil(t, err) + + _, err = master.VttabletProcess.QueryTablet("DROP TABLE vt_insert_test", keyspaceName, true) + assert.Nil(t, err) +} + +// Test a master and replica from the same backup. +// +// Check that a replica and master both restored from the same backup +// can replicate successfully. +func masterReplicaSameBackup(t *testing.T) { + // insert data on master, wait for replica to get it + verifyInitialReplication(t) + + // backup the replica + err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + + // insert more data on the master + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + + // now bring up the other replica, letting it restore from backup. + restoreWaitForBackup(t, "replica") + err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) + assert.Nil(t, err) + + // check the new replica has the data + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + + // Promote replica2 to master + err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", + "-keyspace_shard", shardKsName, + "-new_master", replica2.Alias) + assert.Nil(t, err) + + // insert more data on replica2 (current master) + _, err = replica2.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) + assert.Nil(t, err) + + // Force replica1 to restore from backup. + verifyRestoreTablet(t, replica1, "SERVING") + + // wait for replica1 to catch up. + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 3) + + // This is to test that replicationPosition is processed correctly + // while doing backup/restore after a reparent. + // It is written into the MANIFEST and read back from the MANIFEST. + // + // Take another backup on the replica. + err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + + // Insert more data on replica2 (current master). + _, err = replica2.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test4')", keyspaceName, true) + assert.Nil(t, err) + + // Force replica1 to restore from backup. + verifyRestoreTablet(t, replica1, "SERVING") + + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 4) + err = replica2.VttabletProcess.TearDown() + assert.Nil(t, err) + restartMasterReplica(t) +} + +func restoreOldMasterByRestart(t *testing.T) { + testRestoreOldMaster(t, restoreUsingRestart) +} + +func restoreOldMasterInPlace(t *testing.T) { + testRestoreOldMaster(t, restoreInPlace) +} + +//Test that a former master replicates correctly after being restored. +// +//- Take a backup. +//- Reparent from old master to new master. +//- Force old master to restore from a previous backup using restore_method. +// +//Args: +//restore_method: function accepting one parameter of type tablet.Tablet, +//this function is called to force a restore on the provided tablet +// +func testRestoreOldMaster(t *testing.T, method restoreMethod) { + // insert data on master, wait for replica to get it + verifyInitialReplication(t) + + // backup the replica + err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + + // insert more data on the master + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + + // reparent to replica1 + err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", + "-keyspace_shard", shardKsName, + "-new_master", replica1.Alias) + assert.Nil(t, err) + + // insert more data to new master + _, err = replica1.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) + assert.Nil(t, err) + + // force the old master to restore at the latest backup. + method(t, master) + + // wait for it to catch up. + cluster.VerifyRowsInTablet(t, master, keyspaceName, 3) + + // teardown + restartMasterReplica(t) +} + +func restoreUsingRestart(t *testing.T, tablet *cluster.Vttablet) { + err := tablet.VttabletProcess.TearDown() + assert.Nil(t, err) + verifyRestoreTablet(t, tablet, "SERVING") +} + +func restoreInPlace(t *testing.T, tablet *cluster.Vttablet) { + err := localCluster.VtctlclientProcess.ExecuteCommand("RestoreFromBackup", tablet.Alias) + assert.Nil(t, err) +} + +func restartMasterReplica(t *testing.T) { + // Stop all master, replica tablet and mysql instance + stopAllTablets() + + // remove all backups + backups := listBackups(t) + for _, backup := range backups { + err := localCluster.VtctlclientProcess.ExecuteCommand("RemoveBackup", shardKsName, backup) + assert.Nil(t, err) + } + // start all tablet and mysql instances + var mysqlProcs []*exec.Cmd + for _, tablet := range []*cluster.Vttablet{master, replica1} { + proc, _ := tablet.MysqlctlProcess.StartProcess() + mysqlProcs = append(mysqlProcs, proc) + + err := localCluster.VtctlclientProcess.InitTablet(tablet, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + err = tablet.VttabletProcess.CreateDB(keyspaceName) + assert.Nil(t, err) + err = tablet.VttabletProcess.Setup() + assert.Nil(t, err) + } + for _, proc := range mysqlProcs { + proc.Wait() + } + err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, master.TabletUID) + assert.Nil(t, err) +} + +func stopAllTablets() { + var mysqlProcs []*exec.Cmd + for _, tablet := range []*cluster.Vttablet{master, replica1, replica2} { + tablet.VttabletProcess.TearDown() + proc, _ := tablet.MysqlctlProcess.StopProcess() + mysqlProcs = append(mysqlProcs, proc) + localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) + } + for _, proc := range mysqlProcs { + proc.Wait() + } + for _, tablet := range []*cluster.Vttablet{master, replica1} { + os.RemoveAll(tablet.VttabletProcess.Directory) + } +} + +func terminatedRestore(t *testing.T) { + // insert data on master, wait for replica to get it + verifyInitialReplication(t) + + // backup the replica + err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + + // insert more data on the master + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + + // reparent to replica1 + err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", + "-keyspace_shard", shardKsName, + "-new_master", replica1.Alias) + assert.Nil(t, err) + + // insert more data to new master + _, err = replica1.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) + assert.Nil(t, err) + + terminateRestore(t) + + err = localCluster.VtctlclientProcess.ExecuteCommand("RestoreFromBackup", master.Alias) + assert.Nil(t, err) + + output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", master.Alias) + assert.Nil(t, err) + + var tabletPB topodata.Tablet + err = json.Unmarshal([]byte(output), &tabletPB) + assert.Nil(t, err) + assert.Equal(t, tabletPB.Type, topodata.TabletType_REPLICA) + + _, err = os.Stat(path.Join(master.VttabletProcess.Directory, "restore_in_progress")) + assert.True(t, os.IsNotExist(err)) + + cluster.VerifyRowsInTablet(t, master, keyspaceName, 3) + stopAllTablets() +} + +//test_backup will: +//- create a shard with master and replica1 only +//- run InitShardMaster +//- bring up tablet_replica2 concurrently, telling it to wait for a backup +//- insert some data +//- take a backup +//- insert more data on the master +//- wait for tablet_replica2 to become SERVING +//- check all data is right (before+after backup data) +//- list the backup, remove it +// +//Args: +//tablet_type: 'replica' or 'rdonly'. +// +// +func vtctlBackup(t *testing.T, tabletType string) { + restoreWaitForBackup(t, tabletType) + verifyInitialReplication(t) + + err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + fmt.Println(err) + + backups := listBackups(t) + assert.Equal(t, len(backups), 1) + time.Sleep(10 * time.Second) + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + fmt.Println(err) + + err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) + assert.Nil(t, err) + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + + cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell) + verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups) + + err = replica2.VttabletProcess.TearDown() + assert.Nil(t, err) + + err = localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", replica2.Alias) + assert.Nil(t, err) + _, err = master.VttabletProcess.QueryTablet("DROP TABLE vt_insert_test", keyspaceName, true) + assert.Nil(t, err) + +} + +// This will create schema in master, insert some data to master and verify the same data in replica +func verifyInitialReplication(t *testing.T) { + _, err := master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true) + assert.Nil(t, err) + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test1')", keyspaceName, true) + assert.Nil(t, err) + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 1) +} + +// Bring up another replica concurrently, telling it to wait until a backup +// is available instead of starting up empty. +// +// Override the backup engine implementation to a non-existent one for restore. +// This setting should only matter for taking new backups. We should be able +// to restore a previous backup successfully regardless of this setting. +func restoreWaitForBackup(t *testing.T, tabletType string) { + replica2.Type = tabletType + resetTabletDir(t, replica2) + replicaTabletArgs := commonTabletArg + replicaTabletArgs = append(replicaTabletArgs, "-backup_engine_implementation", "fake_implementation") + replicaTabletArgs = append(replicaTabletArgs, "-wait_for_backup_interval", "1s") + replicaTabletArgs = append(replicaTabletArgs, "-init_tablet_type", tabletType) + replica2.VttabletProcess.ExtraArgs = replicaTabletArgs + replica2.VttabletProcess.ServingStatus = "" + err := replica2.VttabletProcess.Setup() + assert.Nil(t, err) +} + +func resetTabletDir(t *testing.T, tablet *cluster.Vttablet) { + err := cluster.ResetTabletDirectory(*tablet) + assert.Nil(t, err) +} + +func listBackups(t *testing.T) []string { + output, err := localCluster.ListBackups(shardKsName) + assert.Nil(t, err) + return output +} + +func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []string) { + // Remove the backup + for _, backup := range backups { + err := localCluster.VtctlclientProcess.ExecuteCommand("RemoveBackup", shardKsName, backup) + assert.Nil(t, err) + } + + // Now, there should not be no backup + backups = listBackups(t) + assert.Equal(t, len(backups), 0) +} + +func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string) { + err := tablet.VttabletProcess.TearDown() + assert.Nil(t, err) + + resetTabletDir(t, tablet) + tablet.VttabletProcess.ServingStatus = "" + err = tablet.VttabletProcess.Setup() + assert.Nil(t, err) + if status != "" { + err = tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{status}, 25*time.Second) + assert.Nil(t, err) + } + + if tablet.Type == "replica" { + verifyReplicationStatus(t, tablet, "ON") + } else if tablet.Type == "rdonly" { + verifyReplicationStatus(t, tablet, "OFF") + } +} + +func verifyReplicationStatus(t *testing.T, vttablet *cluster.Vttablet, expectedStatus string) { + status, err := vttablet.VttabletProcess.GetDBVar("rpl_semi_sync_slave_enabled", keyspaceName) + assert.Nil(t, err) + assert.Equal(t, status, expectedStatus) + status, err = vttablet.VttabletProcess.GetDBStatus("rpl_semi_sync_slave_status", keyspaceName) + assert.Nil(t, err) + assert.Equal(t, status, expectedStatus) +} + +func terminateRestore(t *testing.T) { + stopRestoreMsg := "Copying file 10" + args := append([]string{"-server", localCluster.VtctlclientProcess.Server, "-alsologtostderr"}, "RestoreFromBackup", master.Alias) + tmpProcess := exec.Command( + "vtctlclient", + args..., + ) + + reader, _ := tmpProcess.StderrPipe() + err := tmpProcess.Start() + assert.Nil(t, err) + found := false + + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + text := scanner.Text() + if strings.Contains(text, stopRestoreMsg) { + if _, err := os.Stat(path.Join(master.VttabletProcess.Directory, "restore_in_progress")); os.IsNotExist(err) { + assert.Fail(t, "restore in progress file missing") + } + tmpProcess.Process.Signal(syscall.SIGTERM) + found = true + return + } + } + assert.True(t, found, "Restore message not found") +} diff --git a/go/test/endtoend/backup/vtctlbackup/main_test.go b/go/test/endtoend/backup/vtctlbackup/main_test.go deleted file mode 100644 index 18ed385c94e..00000000000 --- a/go/test/endtoend/backup/vtctlbackup/main_test.go +++ /dev/null @@ -1,151 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vtctlbackup - -import ( - "flag" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path" - "testing" - - "vitess.io/vitess/go/test/endtoend/cluster" - "vitess.io/vitess/go/test/endtoend/sharding/initialsharding" - "vitess.io/vitess/go/vt/log" -) - -var ( - master *cluster.Vttablet - replica1 *cluster.Vttablet - replica2 *cluster.Vttablet - localCluster *cluster.LocalProcessCluster - newInitDBFile string - cell = cluster.DefaultCell - hostname = "localhost" - keyspaceName = "ks" - dbPassword = "VtDbaPass" - shardKsName = fmt.Sprintf("%s/%s", keyspaceName, shardName) - dbCredentialFile string - shardName = "0" - commonTabletArg = []string{ - "-vreplication_healthcheck_topology_refresh", "1s", - "-vreplication_healthcheck_retry_delay", "1s", - "-vreplication_retry_delay", "1s", - "-degraded_threshold", "5s", - "-lock_tables_timeout", "5s", - "-watch_replication_stream", - "-enable_replication_reporter", - "-serving_state_grace_period", "1s"} -) - -func TestMain(m *testing.M) { - flag.Parse() - - exitCode, err := func() (int, error) { - localCluster = cluster.NewCluster(cell, hostname) - defer localCluster.Teardown() - - // Start topo server - err := localCluster.StartTopo() - if err != nil { - return 1, err - } - - // Start keyspace - keyspace := &cluster.Keyspace{ - Name: keyspaceName, - } - localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace) - - dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory) - initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) - sql := string(initDb) - newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) - ioutil.WriteFile(newInitDBFile, []byte(sql), 0666) - - extraArgs := []string{"-db-credentials-file", dbCredentialFile} - commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) - - shard := cluster.Shard{ - Name: shardName, - } - - var mysqlProcs []*exec.Cmd - for i := 0; i < 3; i++ { - tabletType := "replica" - if i == 0 { - tabletType = "master" - } - tablet := localCluster.GetVttabletInstance(tabletType, 0, cell) - tablet.VttabletProcess = localCluster.GetVtprocessInstanceFromVttablet(tablet, shard.Name, keyspaceName) - tablet.VttabletProcess.DbPassword = dbPassword - tablet.VttabletProcess.ExtraArgs = commonTabletArg - tablet.VttabletProcess.SupportsBackup = true - tablet.VttabletProcess.EnableSemiSync = true - - tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) - tablet.MysqlctlProcess.InitDBFile = newInitDBFile - tablet.MysqlctlProcess.ExtraArgs = extraArgs - if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { - return 1, err - } else { - mysqlProcs = append(mysqlProcs, proc) - } - shard.Vttablets = append(shard.Vttablets, tablet) - } - for _, proc := range mysqlProcs { - if err := proc.Wait(); err != nil { - return 1, err - } - } - master = shard.Vttablets[0] - replica1 = shard.Vttablets[1] - replica2 = shard.Vttablets[2] - - if err := localCluster.VtctlclientProcess.InitTablet(master, cell, keyspaceName, hostname, shard.Name); err != nil { - return 1, err - } - if err := localCluster.VtctlclientProcess.InitTablet(replica1, cell, keyspaceName, hostname, shard.Name); err != nil { - return 1, err - } - - for _, tablet := range []cluster.Vttablet{*master, *replica1} { - if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil { - return 1, err - } - if err := tablet.VttabletProcess.Setup(); err != nil { - return 1, err - } - } - - if err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shard.Name, cell, master.TabletUID); err != nil { - return 1, err - } - return m.Run(), nil - }() - - if err != nil { - log.Error(err.Error()) - os.Exit(1) - } else { - os.Exit(exitCode) - } - -} diff --git a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go new file mode 100644 index 00000000000..99ab22fbf53 --- /dev/null +++ b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go @@ -0,0 +1,39 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctlbackup + +import ( + "fmt" + "testing" + + backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" +) + +// TestXtraBackup - tests the back using xtrabackup arguments +func TestXtraBackup(t *testing.T) { + code, err := backup.LaunchCluster(true) + fmt.Println(err) + if err != nil { + t.Errorf("setup failed with status code %d", code) + } + + // Run all the backup tests + backup.TestBackup(t) + + // Teardown the cluster + backup.TearDownCluster() +} From de23c53e5d292fed7c96224e2fe153dff9222f6b Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 13:27:49 +0530 Subject: [PATCH 07/50] fix sequencing of cleanup Signed-off-by: Ajeet jain --- .../backup/vtctlbackup/backup_utils.go | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 2164e635b2c..6a5a03175f6 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -58,16 +58,15 @@ var ( "-lock_tables_timeout", "5s", "-watch_replication_stream", "-enable_replication_reporter", - "-serving_state_grace_period", "1s"} + "-serving_state_grace_period", "1s", + } xtrabackupArgs = []string{ - "-backup_engine_implementation", - "xtrabackup", + "-backup_engine_implementation", "xtrabackup", "-xtrabackup_stream_mode", "tar", "-xtrabackup_user=vt_dba", "-xtrabackup_stripes=0", - "-xtrabackup_backup_flags", - fmt.Sprintf("--password=%s", dbPassword)} - + "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), + } vtInsertTest = ` create table vt_insert_test ( id bigint auto_increment, @@ -384,7 +383,11 @@ func restartMasterReplica(t *testing.T) { for _, tablet := range []*cluster.Vttablet{master, replica1} { proc, _ := tablet.MysqlctlProcess.StartProcess() mysqlProcs = append(mysqlProcs, proc) - + } + for _, proc := range mysqlProcs { + proc.Wait() + } + for _, tablet := range []*cluster.Vttablet{master, replica1} { err := localCluster.VtctlclientProcess.InitTablet(tablet, cell, keyspaceName, hostname, shardName) assert.Nil(t, err) err = tablet.VttabletProcess.CreateDB(keyspaceName) @@ -392,9 +395,6 @@ func restartMasterReplica(t *testing.T) { err = tablet.VttabletProcess.Setup() assert.Nil(t, err) } - for _, proc := range mysqlProcs { - proc.Wait() - } err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, master.TabletUID) assert.Nil(t, err) } @@ -478,14 +478,12 @@ func vtctlBackup(t *testing.T, tabletType string) { err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) assert.Nil(t, err) - fmt.Println(err) backups := listBackups(t) assert.Equal(t, len(backups), 1) - time.Sleep(10 * time.Second) + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) assert.Nil(t, err) - fmt.Println(err) err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) assert.Nil(t, err) From 769d6f7968bcceb376bdb4b02659a8a6ff99ad3b Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 13:42:46 +0530 Subject: [PATCH 08/50] fix terminate restore for xtrabackup Signed-off-by: Ajeet jain --- go/test/endtoend/backup/vtctlbackup/backup_utils.go | 7 +++++++ go/test/endtoend/backup/xtrabackup/xtrabackup_test.go | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 6a5a03175f6..584563e3819 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -43,6 +43,7 @@ var ( replica2 *cluster.Vttablet localCluster *cluster.LocalProcessCluster newInitDBFile string + useXtrabackup bool cell = cluster.DefaultCell hostname = "localhost" keyspaceName = "ks" @@ -104,6 +105,7 @@ func LaunchCluster(xtrabackup bool) (int, error) { commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) if xtrabackup { + useXtrabackup = xtrabackup commonTabletArg = append(commonTabletArg, xtrabackupArgs...) } @@ -584,6 +586,10 @@ func verifyReplicationStatus(t *testing.T, vttablet *cluster.Vttablet, expectedS func terminateRestore(t *testing.T) { stopRestoreMsg := "Copying file 10" + if useXtrabackup { + stopRestoreMsg = "Restore: Preparing" + } + args := append([]string{"-server", localCluster.VtctlclientProcess.Server, "-alsologtostderr"}, "RestoreFromBackup", master.Alias) tmpProcess := exec.Command( "vtctlclient", @@ -596,6 +602,7 @@ func terminateRestore(t *testing.T) { found := false scanner := bufio.NewScanner(reader) + for scanner.Scan() { text := scanner.Text() if strings.Contains(text, stopRestoreMsg) { diff --git a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go index 99ab22fbf53..16244a01a48 100644 --- a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go +++ b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go @@ -17,7 +17,6 @@ limitations under the License. package vtctlbackup import ( - "fmt" "testing" backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" @@ -26,7 +25,6 @@ import ( // TestXtraBackup - tests the back using xtrabackup arguments func TestXtraBackup(t *testing.T) { code, err := backup.LaunchCluster(true) - fmt.Println(err) if err != nil { t.Errorf("setup failed with status code %d", code) } From 621c848018a9f96f6fdb50d39f4ec9b458b4d208 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 15:02:57 +0530 Subject: [PATCH 09/50] updated config for xtrabackup Signed-off-by: Ajeet jain --- .github/workflows/cluster_endtoend.yml | 6 +++++- test/config.json | 18 +++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index 93fb133d8bd..399cf902ede 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - name: [11, 12, 13, 14, 15, 16, 17] + name: [11, 12, 13, 14, 15, 16, 17, 19] steps: - name: Set up Go @@ -26,6 +26,10 @@ jobs: sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 - name: sharded cluster_endtoend run: | diff --git a/test/config.json b/test/config.json index da04a180222..35ad397ba61 100644 --- a/test/config.json +++ b/test/config.json @@ -198,15 +198,6 @@ "worker_test" ] }, - "xtrabackup": { - "File": "xtrabackup.py", - "Args": [], - "Command": [], - "Manual": false, - "Shard": 1, - "RetryMax": 0, - "Tags": [] - }, "xb_recovery": { "File": "xb_recovery.py", "Args": [], @@ -243,6 +234,15 @@ "RetryMax": 0, "Tags": [] }, + "backup_xtrabackup": { + "File": "xtrabackup.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backup/xtrabackup"], + "Command": [], + "Manual": false, + "Shard": 19, + "RetryMax": 0, + "Tags": [] + }, "binlog": { "File": "binlog.go", "Args": ["vitess.io/vitess/go/test/endtoend/binlog"], From cad5498774e51f8bfb2faca1b360b79da6e88bb7 Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Tue, 21 Jan 2020 15:11:41 +0530 Subject: [PATCH 10/50] backup-mysqlctld: mysqlctld setup module created. Signed-off-by: pradip parmar --- go/test/endtoend/cluster/cluster_process.go | 11 +- go/test/endtoend/cluster/etcd_process.go | 2 +- go/test/endtoend/cluster/mysqlctl_process.go | 21 +- go/test/endtoend/cluster/mysqlctld_process.go | 196 ++++++++++++++++++ go/test/endtoend/cluster/vtctld_process.go | 2 +- go/test/endtoend/cluster/vtgate_process.go | 2 +- go/test/endtoend/cluster/vttablet_process.go | 2 +- go/test/endtoend/cluster/vtworker_process.go | 2 +- go/test/endtoend/mysqlctl/mysqlctl_test.go | 8 +- go/test/endtoend/mysqlctld/mysqlctld_test.go | 164 +++++++++++++++ 10 files changed, 386 insertions(+), 24 deletions(-) create mode 100644 go/test/endtoend/cluster/mysqlctld_process.go create mode 100644 go/test/endtoend/mysqlctld/mysqlctld_test.go diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index 99fe4e835bf..84b9217d323 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -129,8 +129,9 @@ type Vttablet struct { Cell string // background executable processes - MysqlctlProcess MysqlctlProcess - VttabletProcess *VttabletProcess + MysqlctlProcess MysqlctlProcess + MysqlctldProcess MysqlctldProcess + VttabletProcess *VttabletProcess } // StartTopo starts topology server @@ -462,6 +463,12 @@ func (cluster *LocalProcessCluster) Teardown() { mysqlctlProcessList = append(mysqlctlProcessList, proc) } } + if tablet.MysqlctldProcess.TabletUID > 0 { + if err := tablet.MysqlctldProcess.Stop(); err != nil { + log.Errorf("Error in mysqlctl teardown - %s", err.Error()) + } + } + if err := tablet.VttabletProcess.TearDown(); err != nil { log.Errorf("Error in vttablet teardown - %s", err.Error()) } diff --git a/go/test/endtoend/cluster/etcd_process.go b/go/test/endtoend/cluster/etcd_process.go index 98d274105d6..2df85281b86 100644 --- a/go/test/endtoend/cluster/etcd_process.go +++ b/go/test/endtoend/cluster/etcd_process.go @@ -76,6 +76,7 @@ func (etcd *EtcdProcess) Setup() (err error) { etcd.exit = make(chan error) go func() { etcd.exit <- etcd.proc.Wait() + etcd.proc = nil }() timeout := time.Now().Add(60 * time.Second) @@ -117,7 +118,6 @@ func (etcd *EtcdProcess) TearDown(Cell string, originalVtRoot string, currentRoo case <-time.After(10 * time.Second): etcd.proc.Process.Kill() - etcd.proc = nil return <-etcd.exit } diff --git a/go/test/endtoend/cluster/mysqlctl_process.go b/go/test/endtoend/cluster/mysqlctl_process.go index 7a90992aa00..3c2968929f3 100644 --- a/go/test/endtoend/cluster/mysqlctl_process.go +++ b/go/test/endtoend/cluster/mysqlctl_process.go @@ -56,11 +56,11 @@ func (mysqlctl *MysqlctlProcess) InitDb() (err error) { // Start executes mysqlctl command to start mysql instance func (mysqlctl *MysqlctlProcess) Start() (err error) { - if tmpProcess, err := mysqlctl.StartProcess(); err != nil { + tmpProcess, err := mysqlctl.StartProcess() + if err != nil { return err - } else { - return tmpProcess.Wait() } + return tmpProcess.Wait() } // StartProcess starts the mysqlctl and returns the process reference @@ -86,11 +86,11 @@ func (mysqlctl *MysqlctlProcess) StartProcess() (*exec.Cmd, error) { // Stop executes mysqlctl command to stop mysql instance func (mysqlctl *MysqlctlProcess) Stop() (err error) { - if tmpProcess, err := mysqlctl.StopProcess(); err != nil { + tmpProcess, err := mysqlctl.StopProcess() + if err != nil { return err - } else { - return tmpProcess.Wait() } + return tmpProcess.Wait() } // StopProcess executes mysqlctl command to stop mysql instance and returns process reference @@ -133,11 +133,7 @@ func MysqlCtlProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) // StartMySQL starts mysqlctl process func StartMySQL(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) error { tablet.MysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) - err := tablet.MysqlctlProcess.Start() - if err != nil { - return err - } - return nil + return tablet.MysqlctlProcess.Start() } // StartMySQLAndGetConnection create a connection to tablet mysql @@ -152,8 +148,7 @@ func StartMySQLAndGetConnection(ctx context.Context, tablet *Vttablet, username UnixSocket: path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tablet.TabletUID), "/mysql.sock"), } - conn, err := mysql.Connect(ctx, ¶ms) - return conn, err + return mysql.Connect(ctx, ¶ms) } // ExecuteCommandWithOutput executes any mysqlctl command and returns output diff --git a/go/test/endtoend/cluster/mysqlctld_process.go b/go/test/endtoend/cluster/mysqlctld_process.go new file mode 100644 index 00000000000..b8bf6f5c068 --- /dev/null +++ b/go/test/endtoend/cluster/mysqlctld_process.go @@ -0,0 +1,196 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "strings" + "syscall" + "time" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/log" +) + +// MysqlctldProcess is a generic handle for a running mysqlctld command . +// It can be spawned manually +type MysqlctldProcess struct { + Name string + Binary string + LogDirectory string + TabletUID int + MySQLPort int + InitDBFile string + ExtraArgs []string + InitMysql bool + proc *exec.Cmd + exit chan error +} + +// InitDb executes mysqlctld command to add cell info +func (mysqlctld *MysqlctldProcess) InitDb() (err error) { + tmpProcess := exec.Command( + mysqlctld.Binary, + "-log_dir", mysqlctld.LogDirectory, + "-tablet_uid", fmt.Sprintf("%d", mysqlctld.TabletUID), + "-mysql_port", fmt.Sprintf("%d", mysqlctld.MySQLPort), + "-init_db_sql_file", mysqlctld.InitDBFile, + ) + return tmpProcess.Run() +} + +// Start starts the mysqlctld and returns the error. +func (mysqlctld *MysqlctldProcess) Start() error { + _ = createDirectory(mysqlctld.LogDirectory, 0700) + mysqlctld.proc = exec.Command( + mysqlctld.Binary, + "-log_dir", mysqlctld.LogDirectory, + "-tablet_uid", fmt.Sprintf("%d", mysqlctld.TabletUID), + "-mysql_port", fmt.Sprintf("%d", mysqlctld.MySQLPort), + ) + + mysqlctld.proc.Args = append(mysqlctld.proc.Args, mysqlctld.ExtraArgs...) + + if mysqlctld.InitMysql { + mysqlctld.proc.Args = append(mysqlctld.proc.Args, + "-init_db_sql_file", mysqlctld.InitDBFile) + } + + errFile, _ := os.Create(path.Join(mysqlctld.LogDirectory, "mysqlctld-stderr.txt")) + mysqlctld.proc.Stderr = errFile + + mysqlctld.proc.Env = append(mysqlctld.proc.Env, os.Environ()...) + mysqlctld.proc.Stdout = os.Stdout + mysqlctld.proc.Stderr = os.Stderr + + log.Infof("%v %v", strings.Join(mysqlctld.proc.Args, " ")) + + err := mysqlctld.proc.Start() + if err != nil { + return err + } + + mysqlctld.exit = make(chan error) + go func() { + mysqlctld.exit <- mysqlctld.proc.Wait() + mysqlctld.proc = nil + }() + + timeout := time.Now().Add(60 * time.Second) + for time.Now().Before(timeout) { + if err := healthCheck(context.Background(), mysqlctld.TabletUID); err == nil { + return nil + } + select { + case err := <-mysqlctld.exit: + return fmt.Errorf("process '%s' exited prematurely (err: %s)", mysqlctld.Name, err) + default: + time.Sleep(300 * time.Millisecond) + } + } + + return fmt.Errorf("process '%s' timed out after 60s (err: %s)", mysqlctld.Name, <-mysqlctld.exit) + +} + +// Stop executes mysqlctld command to stop mysql instance +func (mysqlctld *MysqlctldProcess) Stop() (err error) { + if mysqlctld.proc == nil || mysqlctld.exit == nil { + return nil + } + + // Attempt graceful shutdown with SIGTERM first + mysqlctld.proc.Process.Signal(syscall.SIGTERM) + // mysqlctld.proc.Process.Signal(syscall.SIGTERM) + + select { + case err := <-mysqlctld.exit: + mysqlctld.proc = nil + return err + + case <-time.After(15 * time.Second): + mysqlctld.proc.Process.Kill() + return <-mysqlctld.exit + } +} + +// CleanupFiles clean the mysql files to make sure we can start the same process again +func (mysqlctld *MysqlctldProcess) CleanupFiles(tabletUID int) { + os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tabletUID))) +} + +// MysqlctldProcessInstance returns a Mysqlctld handle for mysqlctld process +// configured with the given Config. +func MysqlctldProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) *MysqlctldProcess { + mysqlctld := &MysqlctldProcess{ + Name: "mysqlctld", + Binary: "mysqlctld", + LogDirectory: tmpDirectory, + InitDBFile: path.Join(os.Getenv("VTROOT"), "/config/init_db.sql"), + } + mysqlctld.MySQLPort = mySQLPort + mysqlctld.TabletUID = tabletUID + mysqlctld.InitMysql = true + return mysqlctld +} + +// StartMySQLctld starts mysqlctld process +func StartMySQLctld(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) error { + tablet.MysqlctldProcess = *MysqlctldProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) + return tablet.MysqlctldProcess.Start() +} + +// StartMySQLctldAndGetConnection create a connection to tablet mysql +func StartMySQLctldAndGetConnection(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) (*mysql.Conn, error) { + tablet.MysqlctldProcess = *MysqlctldProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) + err := tablet.MysqlctldProcess.Start() + if err != nil { + return nil, err + } + + params := mysql.ConnParams{ + Uname: username, + UnixSocket: path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tablet.TabletUID), "/mysql.sock"), + } + + return mysql.Connect(ctx, ¶ms) +} + +// ExecuteCommandWithOutput executes any mysqlctld command and returns output +func (mysqlctld *MysqlctldProcess) ExecuteCommandWithOutput(args ...string) (result string, err error) { + tmpProcess := exec.Command( + mysqlctld.Binary, + args..., + ) + log.Info(fmt.Sprintf("Executing mysqlctld with arguments %v", strings.Join(tmpProcess.Args, " "))) + resultByte, err := tmpProcess.CombinedOutput() + return string(resultByte), err +} + +func healthCheck(ctx context.Context, tabletUID int) error { + params := mysql.ConnParams{ + Uname: "vt_dba", + UnixSocket: path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tabletUID), "/mysql.sock"), + } + + _, err := mysql.Connect(ctx, ¶ms) + return err +} diff --git a/go/test/endtoend/cluster/vtctld_process.go b/go/test/endtoend/cluster/vtctld_process.go index 89a3ef799d6..4bb4ef1bd99 100644 --- a/go/test/endtoend/cluster/vtctld_process.go +++ b/go/test/endtoend/cluster/vtctld_process.go @@ -87,6 +87,7 @@ func (vtctld *VtctldProcess) Setup(cell string, extraArgs ...string) (err error) vtctld.exit = make(chan error) go func() { vtctld.exit <- vtctld.proc.Wait() + vtctld.proc = nil }() timeout := time.Now().Add(60 * time.Second) @@ -140,7 +141,6 @@ func (vtctld *VtctldProcess) TearDown() error { case <-time.After(10 * time.Second): vtctld.proc.Process.Kill() - vtctld.proc = nil return <-vtctld.exit } } diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index 12dad980510..f9c6857b1d9 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -100,6 +100,7 @@ func (vtgate *VtgateProcess) Setup() (err error) { go func() { if vtgate.proc != nil { vtgate.exit <- vtgate.proc.Wait() + vtgate.proc = nil } }() @@ -194,7 +195,6 @@ func (vtgate *VtgateProcess) TearDown() error { case <-time.After(10 * time.Second): vtgate.proc.Process.Kill() - vtgate.proc = nil return <-vtgate.exit } } diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index 1b70264b669..89cf94c07ce 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -125,6 +125,7 @@ func (vttablet *VttabletProcess) Setup() (err error) { go func() { if vttablet.proc != nil { vttablet.exit <- vttablet.proc.Wait() + vttablet.proc = nil } }() @@ -281,7 +282,6 @@ func (vttablet *VttabletProcess) TearDown() error { case <-time.After(10 * time.Second): vttablet.proc.Process.Kill() - vttablet.proc = nil return <-vttablet.exit } } diff --git a/go/test/endtoend/cluster/vtworker_process.go b/go/test/endtoend/cluster/vtworker_process.go index 46ab0fb715b..3fb600655ed 100644 --- a/go/test/endtoend/cluster/vtworker_process.go +++ b/go/test/endtoend/cluster/vtworker_process.go @@ -85,6 +85,7 @@ func (vtworker *VtworkerProcess) Setup(cell string) (err error) { vtworker.exit = make(chan error) go func() { vtworker.exit <- vtworker.proc.Wait() + vtworker.proc = nil }() timeout := time.Now().Add(60 * time.Second) @@ -131,7 +132,6 @@ func (vtworker *VtworkerProcess) TearDown() error { case <-time.After(10 * time.Second): vtworker.proc.Process.Kill() - vtworker.proc = nil return <-vtworker.exit } } diff --git a/go/test/endtoend/mysqlctl/mysqlctl_test.go b/go/test/endtoend/mysqlctl/mysqlctl_test.go index b5f70954eb9..c763e777482 100644 --- a/go/test/endtoend/mysqlctl/mysqlctl_test.go +++ b/go/test/endtoend/mysqlctl/mysqlctl_test.go @@ -43,7 +43,7 @@ func TestMain(m *testing.M) { flag.Parse() exitCode := func() int { - clusterInstance = &cluster.LocalProcessCluster{Cell: cell, Hostname: hostname} + clusterInstance = cluster.NewCluster(cell, hostname) defer clusterInstance.Teardown() // Start topo server @@ -97,11 +97,11 @@ func initCluster(shardNames []string, totalTabletsRequired int) { } // Start Mysqlctl process tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, clusterInstance.TmpDirectory) - if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { + proc, err := tablet.MysqlctlProcess.StartProcess() + if err != nil { return - } else { - mysqlCtlProcessList = append(mysqlCtlProcessList, proc) } + mysqlCtlProcessList = append(mysqlCtlProcessList, proc) // start vttablet process tablet.VttabletProcess = cluster.VttabletProcessInstance(tablet.HTTPPort, diff --git a/go/test/endtoend/mysqlctld/mysqlctld_test.go b/go/test/endtoend/mysqlctld/mysqlctld_test.go new file mode 100644 index 00000000000..3624aab4788 --- /dev/null +++ b/go/test/endtoend/mysqlctld/mysqlctld_test.go @@ -0,0 +1,164 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysqlctld + +import ( + "flag" + "fmt" + "os" + "testing" + + "vitess.io/vitess/go/vt/log" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + masterTablet cluster.Vttablet + replicaTablet cluster.Vttablet + hostname = "localhost" + keyspaceName = "test_keyspace" + shardName = "0" + cell = "zone1" +) + +func TestMain(m *testing.M) { + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(cell, hostname) + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + return 1 + } + + if err := clusterInstance.VtctlProcess.CreateKeyspace(keyspaceName); err != nil { + return 1 + } + + if err := initCluster([]string{"0"}, 2); err != nil { + return 1 + } + + // Collect table paths and ports + tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets + for _, tablet := range tablets { + if tablet.Type == "master" { + masterTablet = *tablet + } else if tablet.Type != "rdonly" { + replicaTablet = *tablet + } + } + + return m.Run() + }() + os.Exit(exitCode) +} + +func initCluster(shardNames []string, totalTabletsRequired int) error { + keyspace := cluster.Keyspace{ + Name: keyspaceName, + } + for _, shardName := range shardNames { + shard := &cluster.Shard{ + Name: shardName, + } + for i := 0; i < totalTabletsRequired; i++ { + // instantiate vttablet object with reserved ports + tabletUID := clusterInstance.GetAndReserveTabletUID() + tablet := &cluster.Vttablet{ + TabletUID: tabletUID, + HTTPPort: clusterInstance.GetAndReservePort(), + GrpcPort: clusterInstance.GetAndReservePort(), + MySQLPort: clusterInstance.GetAndReservePort(), + Alias: fmt.Sprintf("%s-%010d", clusterInstance.Cell, tabletUID), + } + if i == 0 { // Make the first one as master + tablet.Type = "master" + } + // Start Mysqlctld process + tablet.MysqlctldProcess = *cluster.MysqlctldProcessInstance(tablet.TabletUID, tablet.MySQLPort, clusterInstance.TmpDirectory) + err := tablet.MysqlctldProcess.Start() + if err != nil { + return err + } + + // start vttablet process + tablet.VttabletProcess = cluster.VttabletProcessInstance(tablet.HTTPPort, + tablet.GrpcPort, + tablet.TabletUID, + clusterInstance.Cell, + shardName, + keyspaceName, + clusterInstance.VtctldProcess.Port, + tablet.Type, + clusterInstance.TopoProcess.Port, + clusterInstance.Hostname, + clusterInstance.TmpDirectory, + clusterInstance.VtTabletExtraArgs, + clusterInstance.EnableSemiSync) + tablet.Alias = tablet.VttabletProcess.TabletPath + + shard.Vttablets = append(shard.Vttablets, tablet) + } + + for _, tablet := range shard.Vttablets { + if _, err := tablet.VttabletProcess.QueryTablet(fmt.Sprintf("create database vt_%s", keyspace.Name), keyspace.Name, false); err != nil { + log.Error(err.Error()) + return err + } + } + + keyspace.Shards = append(keyspace.Shards, *shard) + } + clusterInstance.Keyspaces = append(clusterInstance.Keyspaces, keyspace) + + return nil +} + +func TestRestart(t *testing.T) { + err := masterTablet.MysqlctldProcess.Stop() + assert.Nil(t, err) + masterTablet.MysqlctldProcess.CleanupFiles(masterTablet.TabletUID) + err = masterTablet.MysqlctldProcess.Start() + assert.Nil(t, err) +} + +func TestAutoDetect(t *testing.T) { + + // Start up tablets with an empty MYSQL_FLAVOR, which means auto-detect + sqlFlavor := os.Getenv("MYSQL_FLAVOR") + os.Setenv("MYSQL_FLAVOR", "") + + err := clusterInstance.Keyspaces[0].Shards[0].Vttablets[0].VttabletProcess.Setup() + assert.Nil(t, err, "error should be nil") + err = clusterInstance.Keyspaces[0].Shards[0].Vttablets[1].VttabletProcess.Setup() + assert.Nil(t, err, "error should be nil") + + // Reparent tablets, which requires flavor detection + err = clusterInstance.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, masterTablet.TabletUID) + assert.Nil(t, err, "error should be nil") + + //Reset flavor + os.Setenv("MYSQL_FLAVOR", sqlFlavor) + +} From d556aa321215268598052b8926fed981cf49a92f Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 15:45:34 +0530 Subject: [PATCH 11/50] added xtrabackup stream mode Signed-off-by: Ajeet jain --- .../backup/vtctlbackup/backup_test.go | 2 +- .../backup/vtctlbackup/backup_utils.go | 10 +++-- .../backup/xtrabackup/xtrabackup_test.go | 2 +- .../xtrabackup_stream_test.go | 37 +++++++++++++++++++ 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go diff --git a/go/test/endtoend/backup/vtctlbackup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go index b98435a30c2..f00f793b47d 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -20,7 +20,7 @@ import "testing" // TestBackupMain - main tests backup using vtctl commands func TestBackupMain(t *testing.T) { - code, err := LaunchCluster(false) + code, err := LaunchCluster(false, "", 0) if err != nil { t.Errorf("setup failed with status code %d", code) } diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 584563e3819..40f8e6aa592 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -44,6 +44,8 @@ var ( localCluster *cluster.LocalProcessCluster newInitDBFile string useXtrabackup bool + xbStreamMode string + xbStripes int cell = cluster.DefaultCell hostname = "localhost" keyspaceName = "ks" @@ -63,9 +65,9 @@ var ( } xtrabackupArgs = []string{ "-backup_engine_implementation", "xtrabackup", - "-xtrabackup_stream_mode", "tar", + "-xtrabackup_stream_mode", xbStreamMode, "-xtrabackup_user=vt_dba", - "-xtrabackup_stripes=0", + fmt.Sprintf("-xtrabackup_stripes=%d", xbStripes), "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), } vtInsertTest = ` @@ -76,7 +78,7 @@ var ( ) Engine=InnoDB` ) -func LaunchCluster(xtrabackup bool) (int, error) { +func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error) { localCluster = cluster.NewCluster(cell, hostname) // Start topo server @@ -106,6 +108,8 @@ func LaunchCluster(xtrabackup bool) (int, error) { if xtrabackup { useXtrabackup = xtrabackup + xbStreamMode = streamMode + xbStripes = stripes commonTabletArg = append(commonTabletArg, xtrabackupArgs...) } diff --git a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go index 16244a01a48..c6346bcb428 100644 --- a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go +++ b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go @@ -24,7 +24,7 @@ import ( // TestXtraBackup - tests the back using xtrabackup arguments func TestXtraBackup(t *testing.T) { - code, err := backup.LaunchCluster(true) + code, err := backup.LaunchCluster(true, "tar", 0) if err != nil { t.Errorf("setup failed with status code %d", code) } diff --git a/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go b/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go new file mode 100644 index 00000000000..e25b5e758bc --- /dev/null +++ b/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go @@ -0,0 +1,37 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctlbackup + +import ( + "testing" + + backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" +) + +// TestXtraBackup - tests the back using xtrabackup arguments +func TestXtraBackup(t *testing.T) { + code, err := backup.LaunchCluster(true, "xbstream", 8) + if err != nil { + t.Errorf("setup failed with status code %d", code) + } + + // Run all the backup tests + backup.TestBackup(t) + + // Teardown the cluster + backup.TearDownCluster() +} From 7cd55aa43e37344041aab930099eaaba7c823af2 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 15:51:02 +0530 Subject: [PATCH 12/50] updated config for xtrabackup stream Signed-off-by: Ajeet jain --- .github/workflows/cluster_endtoend.yml | 2 +- .../backup/vtctlbackup/backup_utils.go | 2 +- test/config.json | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index 399cf902ede..66f3b6bc2e0 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - name: [11, 12, 13, 14, 15, 16, 17, 19] + name: [11, 12, 13, 14, 15, 16, 17, 19, 20] steps: - name: Set up Go diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 40f8e6aa592..f6b121454d4 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -65,7 +65,7 @@ var ( } xtrabackupArgs = []string{ "-backup_engine_implementation", "xtrabackup", - "-xtrabackup_stream_mode", xbStreamMode, + fmt.Sprintf("-xtrabackup_stream_mode=%s", xbStreamMode), "-xtrabackup_user=vt_dba", fmt.Sprintf("-xtrabackup_stripes=%d", xbStripes), "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), diff --git a/test/config.json b/test/config.json index 35ad397ba61..ac25ea215a5 100644 --- a/test/config.json +++ b/test/config.json @@ -207,15 +207,6 @@ "RetryMax": 0, "Tags": [] }, - "xtrabackup_xbstream": { - "File": "xtrabackup_xbstream.py", - "Args": [], - "Command": [], - "Manual": false, - "Shard": 3, - "RetryMax": 0, - "Tags": [] - }, "backup": { "File": "backup.go", "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtctlbackup"], @@ -243,6 +234,15 @@ "RetryMax": 0, "Tags": [] }, + "backup_xtrabackup_xbstream": { + "File": "xtrabackup_xbstream.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backup/xtrabackupstream"], + "Command": [], + "Manual": false, + "Shard": 20, + "RetryMax": 0, + "Tags": [] + }, "binlog": { "File": "binlog.go", "Args": ["vitess.io/vitess/go/test/endtoend/binlog"], From 502cacb0ec278721e22e9c0cce4595abe6cf56ab Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 15:55:42 +0530 Subject: [PATCH 13/50] minor changes to config Signed-off-by: Ajeet jain --- .../backup/vtctlbackup/backup_utils.go | 18 +++++++++++------- .../backup/xtrabackup/xtrabackup_test.go | 4 ++-- .../xtrabackupstream/xtrabackup_stream_test.go | 4 ++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index f6b121454d4..33c955467ab 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -63,13 +63,7 @@ var ( "-enable_replication_reporter", "-serving_state_grace_period", "1s", } - xtrabackupArgs = []string{ - "-backup_engine_implementation", "xtrabackup", - fmt.Sprintf("-xtrabackup_stream_mode=%s", xbStreamMode), - "-xtrabackup_user=vt_dba", - fmt.Sprintf("-xtrabackup_stripes=%d", xbStripes), - "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), - } + vtInsertTest = ` create table vt_insert_test ( id bigint auto_increment, @@ -106,10 +100,20 @@ func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error) extraArgs := []string{"-db-credentials-file", dbCredentialFile} commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) + // Update arguments for xtrabackup if xtrabackup { useXtrabackup = xtrabackup xbStreamMode = streamMode xbStripes = stripes + + xtrabackupArgs := []string{ + "-backup_engine_implementation", "xtrabackup", + fmt.Sprintf("-xtrabackup_stream_mode=%s", xbStreamMode), + "-xtrabackup_user=vt_dba", + fmt.Sprintf("-xtrabackup_stripes=%d", xbStripes), + "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), + } + commonTabletArg = append(commonTabletArg, xtrabackupArgs...) } diff --git a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go index c6346bcb428..4b94086bcd3 100644 --- a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go +++ b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go @@ -22,8 +22,8 @@ import ( backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" ) -// TestXtraBackup - tests the back using xtrabackup arguments -func TestXtraBackup(t *testing.T) { +// TestXtraBackup - tests the backup using xtrabackup +func TestXtrabackup(t *testing.T) { code, err := backup.LaunchCluster(true, "tar", 0) if err != nil { t.Errorf("setup failed with status code %d", code) diff --git a/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go b/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go index e25b5e758bc..5e86e73719e 100644 --- a/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go +++ b/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go @@ -22,8 +22,8 @@ import ( backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" ) -// TestXtraBackup - tests the back using xtrabackup arguments -func TestXtraBackup(t *testing.T) { +// TestXtrabackupStream - tests the backup using xtrabackup with xbstream mode +func TestXtrabackupStream(t *testing.T) { code, err := backup.LaunchCluster(true, "xbstream", 8) if err != nil { t.Errorf("setup failed with status code %d", code) From bdbc98941d97729a57981a246205808703aeee34 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Fri, 10 Jan 2020 11:47:28 +0530 Subject: [PATCH 14/50] initial commit for backup_only Signed-off-by: Ajeet jain --- .../endtoend/backup_only/backup_only_test.go | 308 ++++++++++++++++++ go/test/endtoend/backup_only/main_test.go | 141 ++++++++ go/test/endtoend/cluster/cluster_process.go | 26 ++ go/test/endtoend/cluster/vtbackup_process.go | 144 ++++++++ go/test/endtoend/cluster/vttablet_process.go | 3 + 5 files changed, 622 insertions(+) create mode 100644 go/test/endtoend/backup_only/backup_only_test.go create mode 100644 go/test/endtoend/backup_only/main_test.go create mode 100644 go/test/endtoend/cluster/vtbackup_process.go diff --git a/go/test/endtoend/backup_only/backup_only_test.go b/go/test/endtoend/backup_only/backup_only_test.go new file mode 100644 index 00000000000..ed7d325fb88 --- /dev/null +++ b/go/test/endtoend/backup_only/backup_only_test.go @@ -0,0 +1,308 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backup_only + +import ( + "bufio" + "fmt" + "os" + "path" + "strings" + "testing" + "time" + + "vitess.io/vitess/go/test/endtoend/cluster" + + "github.com/stretchr/testify/assert" + + "vitess.io/vitess/go/vt/log" +) + +var ( + vtInsertTest = ` + create table vt_insert_test ( + id bigint auto_increment, + msg varchar(64), + primary key (id) + ) Engine=InnoDB;` +) + +func TestTabletInitialBackup(t *testing.T) { + // Test Initial Backup Flow + // TestTabletInitialBackup will: + // - Create a shard using vtbackup and --initial-backup + // - Create the rest of the cluster restoring from backup + // - Externally Reparenting to a master tablet + // - Insert Some data + // - Verify that the cluster is working + // - Take a Second Backup + // - Bring up a second replica, and restore from the second backup + // - list the backups, remove them + + vtBackup(t, true) + backups := countBackups(t) + assert.Equal(t, 1, backups) + + // Initialize the tablets + initTablets(t, false, false) + + // Restore the Tablets + restore(t, master, "replica", "NOT_SERVING") + err := localCluster.VtctlclientProcess.ExecuteCommand( + "TabletExternallyReparented", master.Alias) + assert.Nil(t, err) + restore(t, replica1, "replica", "SERVING") + + // Run the entire backup test + firstBackupTest(t, "replica") + +} +func TestTabletBackupOnly(t *testing.T) { + // Test Backup Flow + // TestTabletBackupOnly will: + // - Create a shard using regular init & start tablet + // - Run initShardMaster to start replication + // - Insert Some data + // - Verify that the cluster is working + // - Take a Second Backup + // - Bring up a second replica, and restore from the second backup + // - list the backups, remove them + + // Reset the tablet object values in order on init tablet in the next step. + master.VttabletProcess.ServingStatus = "NOT_SERVING" + replica1.VttabletProcess.ServingStatus = "NOT_SERVING" + + initTablets(t, true, true) + firstBackupTest(t, "replica") +} + +func firstBackupTest(t *testing.T, tabletType string) { + // Test First Backup flow. + // + // firstBackupTest will: + // - create a shard with master and replica1 only + // - run InitShardMaster + // - insert some data + // - take a backup + // - insert more data on the master + // - bring up replica2 after the fact, let it restore the backup + // - check all data is right (before+after backup data) + // - list the backup, remove it + + // Store initial backup counts + backupsCount := countBackups(t) + + // insert data on master, wait for slave to get it + _, err := master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true) + assert.Nil(t, err) + // Add a single row with value 'test1' to the master tablet + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test1')", keyspaceName, true) + assert.Nil(t, err) + + // Check that the specified tablet has the expected number of rows + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 1) + + // backup the slave + log.Info("taking backup %s", time.Now()) + vtBackup(t, false) + log.Info("done taking backup %s", time.Now()) + + // check that the backup shows up in the listing + backups := countBackups(t) + assert.Equal(t, backups, backupsCount+1) + + // insert more data on the master + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 2) + + // now bring up the other slave, letting it restore from backup. + err = localCluster.VtctlclientProcess.InitTablet(replica2, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + restore(t, replica2, "replica", "SERVING") + // Replica2 takes time to server. Sleeping for 5 sec. + time.Sleep(5 * time.Second) + //check the new slave has the data + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + + // check that the restored slave has the right local_metadata + result, err := replica2.VttabletProcess.QueryTabletWithDB("select * from local_metadata", "_vt") + assert.Nil(t, err) + assert.Equal(t, replica2.Alias, result.Rows[0][1].ToString(), "Alias") + assert.Equal(t, "ks.0", result.Rows[1][1].ToString(), "ClusterAlias") + assert.Equal(t, cell, result.Rows[2][1].ToString(), "DataCenter") + if tabletType == "replica" { + assert.Equal(t, "neutral", result.Rows[3][1].ToString(), "PromotionRule") + } else { + assert.Equal(t, "must_not", result.Rows[3][1].ToString(), "PromotionRule") + } + + removeBackups(t) + backups = countBackups(t) + assert.Equal(t, 0, backups) + + tearDown(t) +} + +func vtBackup(t *testing.T, initialBackup bool) { + // Take the back using vtbackup executable + extraArgs := []string{"-allow_first_backup", "-db-credentials-file", dbCredentialFile} + log.Info("starting backup tablet %s", time.Now()) + err := localCluster.StartVtbackup(newInitDBFile, initialBackup, keyspaceName, shardName, cell, extraArgs...) + assert.Nil(t, err) +} + +func listBackups(t *testing.T) string { + // Get a list of backup names for the current shard. + localCluster.VtctlProcess = *cluster.VtctlProcessInstance(localCluster.TopoPort, localCluster.Hostname) + backups, err := localCluster.VtctlProcess.ExecuteCommandWithOutput( + "-backup_storage_implementation", "file", + "-file_backup_storage_root", + path.Join(os.Getenv("VTDATAROOT"), "tmp", "backupstorage"), + "ListBackups", shardKsName, + ) + assert.Nil(t, err) + return backups +} + +func countBackups(t *testing.T) int { + // Count the number of backups available in current shard. + backupList := listBackups(t) + backupCount := 0 + // Counts the available backups + scanner := bufio.NewScanner(strings.NewReader(backupList)) + for scanner.Scan() { + if scanner.Text() != "" { + backupCount++ + } + } + return backupCount +} + +func removeBackups(t *testing.T) { + // Remove all the backups from the shard + backupList := listBackups(t) + + scanner := bufio.NewScanner(strings.NewReader(backupList)) + for scanner.Scan() { + if scanner.Text() != "" { + _, err := localCluster.VtctlProcess.ExecuteCommandWithOutput( + "-backup_storage_implementation", "file", + "-file_backup_storage_root", + path.Join(os.Getenv("VTDATAROOT"), "tmp", "backupstorage"), + "RemoveBackup", shardKsName, scanner.Text(), + ) + assert.Nil(t, err) + } + } + +} + +func initTablets(t *testing.T, startTablet bool, initShardMaster bool) { + // Initialize tablets + for _, tablet := range []cluster.Vttablet{*master, *replica1} { + err := localCluster.VtctlclientProcess.InitTablet(&tablet, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + if startTablet { + err = tablet.VttabletProcess.Setup() + assert.Nil(t, err) + } + } + + if initShardMaster { + // choose master and start replication + err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, master.TabletUID) + + if err != nil { + fmt.Println("*********______________****************") + fmt.Println(err) + fmt.Println(master.HTTPPort) + fmt.Println(replica1.HTTPPort) + time.Sleep(5 * time.Minute) + } + assert.Nil(t, err) + } +} + +func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForState string) { + // Erase mysql/tablet dir, then start tablet with restore enabled. + + log.Info("restoring tablet %s", time.Now()) + resetTabletDirectory(t, *tablet) + + err := tablet.VttabletProcess.CreateDB(keyspaceName) + assert.Nil(t, err) + + // Start tablets + tablet.VttabletProcess.ExtraArgs = []string{"-db-credentials-file", dbCredentialFile} + tablet.VttabletProcess.TabletType = tabletType + tablet.VttabletProcess.ServingStatus = waitForState + tablet.VttabletProcess.SupportsBackup = true + err = tablet.VttabletProcess.Setup() + assert.Nil(t, err) +} + +func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet) { + + extraArgs := []string{"-db-credentials-file", dbCredentialFile} + tablet.MysqlctlProcess.ExtraArgs = extraArgs + + // Shutdown Mysql + err := tablet.MysqlctlProcess.Stop() + assert.Nil(t, err) + // Teardown Tablet + err = tablet.VttabletProcess.TearDown() + assert.Nil(t, err) + + // Empty the dir + err = os.RemoveAll(tablet.VttabletProcess.Directory) + assert.Nil(t, err) + + // Init the Mysql + tablet.MysqlctlProcess.InitDBFile = newInitDBFile + err = tablet.MysqlctlProcess.Start() + assert.Nil(t, err) +} + +func tearDown(t *testing.T) { + for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { + //Tear down Tablet + err := tablet.VttabletProcess.TearDown() + assert.Nil(t, err) + err = localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) + assert.Nil(t, err) + + //resetTabletDirectory(t, tablet) + } + + // reset replication + promoteSlaveCommands := "STOP SLAVE; RESET SLAVE ALL; RESET MASTER;" + disableSemiSyncCommands := "SET GLOBAL rpl_semi_sync_master_enabled = false; SET GLOBAL rpl_semi_sync_slave_enabled = false" + for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { + _, err := tablet.VttabletProcess.QueryTablet(promoteSlaveCommands, keyspaceName, true) + assert.Nil(t, err) + _, err = tablet.VttabletProcess.QueryTablet(disableSemiSyncCommands, keyspaceName, true) + assert.Nil(t, err) + + for _, db := range []string{"_vt", "vt_insert_test"} { + _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("drop database if exists %s", db), keyspaceName, true) + assert.Nil(t, err) + } + } + +} diff --git a/go/test/endtoend/backup_only/main_test.go b/go/test/endtoend/backup_only/main_test.go new file mode 100644 index 00000000000..e4dc4b83595 --- /dev/null +++ b/go/test/endtoend/backup_only/main_test.go @@ -0,0 +1,141 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backup_only + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "testing" + + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/sharding/initialsharding" + "vitess.io/vitess/go/vt/log" +) + +var ( + master *cluster.Vttablet + replica1 *cluster.Vttablet + replica2 *cluster.Vttablet + localCluster *cluster.LocalProcessCluster + newInitDBFile string + cell = cluster.DefaultCell + hostname = "localhost" + keyspaceName = "ks" + shardName = "0" + dbPassword = "VtDbaPass" + shardKsName = fmt.Sprintf("%s/%s", keyspaceName, shardName) + dbCredentialFile string + commonTabletArg = []string{ + "-vreplication_healthcheck_topology_refresh", "1s", + "-vreplication_healthcheck_retry_delay", "1s", + "-vreplication_retry_delay", "1s", + "-degraded_threshold", "5s", + "-lock_tables_timeout", "5s", + "-watch_replication_stream", + "-enable_replication_reporter", + "-serving_state_grace_period", "1s"} +) + +func TestMain(m *testing.M) { + flag.Parse() + + exitCode, err := func() (int, error) { + localCluster = cluster.NewCluster(cell, hostname) + defer localCluster.Teardown() + + // Start topo server + err := localCluster.StartTopo() + if err != nil { + return 1, err + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + } + localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace) + + // Create a new init_db.sql file that sets up passwords for all users. + // Then we use a db-credentials-file with the passwords. + dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory) + initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) + sql := string(initDb) + newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") + sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + err = ioutil.WriteFile(newInitDBFile, []byte(sql), 0666) + if err != nil { + return 1, err + } + + extraArgs := []string{"-db-credentials-file", dbCredentialFile} + commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) + + shard := cluster.Shard{ + Name: shardName, + } + + master = localCluster.GetVttabletInstance("replica", 0, "") + replica1 = localCluster.GetVttabletInstance("replica", 0, "") + replica2 = localCluster.GetVttabletInstance("replica", 0, "") + shard.Vttablets = []*cluster.Vttablet{master, replica1, replica2} + + // Start MySql processes + var mysqlProcs []*exec.Cmd + for _, tablet := range shard.Vttablets { + tablet.VttabletProcess = localCluster.GetVtprocessInstanceFromVttablet(tablet, shard.Name, keyspaceName) + tablet.VttabletProcess.DbPassword = dbPassword + tablet.VttabletProcess.ExtraArgs = commonTabletArg + tablet.VttabletProcess.SupportsBackup = true + tablet.VttabletProcess.EnableSemiSync = true + + tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) + tablet.MysqlctlProcess.InitDBFile = newInitDBFile + tablet.MysqlctlProcess.ExtraArgs = extraArgs + if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { + return 1, err + } else { + mysqlProcs = append(mysqlProcs, proc) + } + } + for _, proc := range mysqlProcs { + if err := proc.Wait(); err != nil { + return 1, err + } + } + + // Create database + for _, tablet := range []cluster.Vttablet{*master, *replica1} { + if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil { + return 1, err + } + } + + return m.Run(), nil + }() + + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } else { + os.Exit(exitCode) + } + +} diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index b73a37a18c3..680abbb1fa5 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -64,6 +64,7 @@ type LocalProcessCluster struct { VtctldProcess VtctldProcess VtgateProcess VtgateProcess VtworkerProcess VtworkerProcess + VtbackupProcess VtbackupProcess nextPortForProcess int @@ -482,6 +483,11 @@ func (cluster *LocalProcessCluster) Teardown() { if err := cluster.TopoProcess.TearDown(cluster.Cell, cluster.OriginalVTDATAROOT, cluster.CurrentVTDATAROOT, *keepData); err != nil { log.Errorf("Error in etcd teardown - %s", err.Error()) } + + if err := cluster.VtbackupProcess.TearDown(); err != nil { + log.Errorf("Error in VtBackup teardown - %s", err.Error()) + } + } // StartVtworker starts a vtworker @@ -500,6 +506,26 @@ func (cluster *LocalProcessCluster) StartVtworker(cell string, extraArgs ...stri } +// StartVtbackup starts a vtbackup +func (cluster *LocalProcessCluster) StartVtbackup(newInitDBFile string, initalBackup bool, + keyspace string, shard string, cell string, extraArgs ...string) error { + log.Info("Starting vtbackup") + cluster.VtbackupProcess = *VtbackupProcessInstance( + cluster.GetAndReserveTabletUID(), + cluster.GetAndReservePort(), + newInitDBFile, + keyspace, + shard, + cell, + cluster.Hostname, + cluster.TmpDirectory, + cluster.TopoPort, + initalBackup) + cluster.VtbackupProcess.ExtraArgs = extraArgs + return cluster.VtbackupProcess.Setup() + +} + // GetAndReservePort gives port for required process func (cluster *LocalProcessCluster) GetAndReservePort() int { if cluster.nextPortForProcess == 0 { diff --git a/go/test/endtoend/cluster/vtbackup_process.go b/go/test/endtoend/cluster/vtbackup_process.go new file mode 100644 index 00000000000..b8f4d866545 --- /dev/null +++ b/go/test/endtoend/cluster/vtbackup_process.go @@ -0,0 +1,144 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "os" + "os/exec" + "path" + "strings" + "syscall" + "time" + + "vitess.io/vitess/go/vt/log" +) + +// VtbackupProcess is a generic handle for a running Vtbackup. +// It can be spawned manually +type VtbackupProcess struct { + Name string + Binary string + CommonArg VtctlProcess + LogDir string + MysqlPort int + Directory string + + Cell string + Keyspace string + Shard string + TabletAlias string + Server string + + ExtraArgs []string + initialBackup bool + initDBfile string + dbPassword string + dbName string + + proc *exec.Cmd + exit chan error +} + +// Setup starts vtbackup process with required arguements +func (vtbackup *VtbackupProcess) Setup() (err error) { + + vtbackup.proc = exec.Command( + vtbackup.Binary, + "-topo_implementation", vtbackup.CommonArg.TopoImplementation, + "-topo_global_server_address", vtbackup.CommonArg.TopoGlobalAddress, + "-topo_global_root", vtbackup.CommonArg.TopoGlobalRoot, + "-log_dir", vtbackup.LogDir, + + //initDBfile is required to run vtbackup + "-mysql_port", fmt.Sprintf("%d", vtbackup.MysqlPort), + "-init_db_sql_file", vtbackup.initDBfile, + "-init_keyspace", vtbackup.Keyspace, + "-init_shard", vtbackup.Shard, + + //Backup Arguments are not optional + "-backup_storage_implementation", "file", + "-file_backup_storage_root", + path.Join(os.Getenv("VTDATAROOT"), "tmp", "backupstorage"), + ) + + if vtbackup.initialBackup { + vtbackup.proc.Args = append(vtbackup.proc.Args, "-initial_backup") + } + if vtbackup.ExtraArgs != nil { + vtbackup.proc.Args = append(vtbackup.proc.Args, vtbackup.ExtraArgs...) + } + + vtbackup.proc.Stderr = os.Stderr + vtbackup.proc.Stdout = os.Stdout + + vtbackup.proc.Env = append(vtbackup.proc.Env, os.Environ()...) + log.Infof("%v", strings.Join(vtbackup.proc.Args, " ")) + fmt.Println(vtbackup.proc.Args) + + err = vtbackup.proc.Run() + if err != nil { + return + } + + return nil +} + +// TearDown shutdowns the running vtbackup process +func (vtbackup *VtbackupProcess) TearDown() error { + if vtbackup.proc == nil || vtbackup.exit == nil { + return nil + } + + // Attempt graceful shutdown with SIGTERM first + vtbackup.proc.Process.Signal(syscall.SIGTERM) + + select { + case err := <-vtbackup.exit: + vtbackup.proc = nil + return err + + case <-time.After(10 * time.Second): + vtbackup.proc.Process.Kill() + vtbackup.proc = nil + return <-vtbackup.exit + } +} + +// VtbackupProcessInstance returns a vtbackup handle +// configured with the given Config. +// The process must be manually started by calling Setup() + +func VtbackupProcessInstance(tabletUid int, mysqlPort int, newInitDBFile string, keyspace string, shard string, + cell string, hostname string, tmpDirectory string, topoPort int, initialBackup bool) *VtbackupProcess { + vtctl := VtctlProcessInstance(topoPort, hostname) + vtbackup := &VtbackupProcess{ + Name: "vtbackup", + Binary: "vtbackup", + CommonArg: *vtctl, + LogDir: tmpDirectory, + Directory: os.Getenv("VTDATAROOT"), + TabletAlias: fmt.Sprintf("%s-%010d", cell, tabletUid), + initDBfile: newInitDBFile, + Keyspace: keyspace, + Shard: shard, + Cell: cell, + MysqlPort: mysqlPort, + initialBackup: initialBackup, + } + return vtbackup +} diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index 1b70264b669..f4a98715d12 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -319,6 +319,9 @@ func (vttablet *VttabletProcess) QueryTabletWithDB(query string, dbname string) UnixSocket: path.Join(vttablet.Directory, "mysql.sock"), DbName: dbname, } + if vttablet.DbPassword != "" { + dbParams.Pass = vttablet.DbPassword + } return executeQuery(dbParams, query) } From ed44822e6e615866bc7d4dec584720b10efca187 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Fri, 10 Jan 2020 11:56:59 +0530 Subject: [PATCH 15/50] changes in package structure Signed-off-by: Ajeet jain --- .../{backup_only => backup/vtbackup}/backup_only_test.go | 2 +- .../endtoend/{backup_only => backup/vtbackup}/main_test.go | 2 +- go/test/endtoend/cluster/vtbackup_process.go | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) rename go/test/endtoend/{backup_only => backup/vtbackup}/backup_only_test.go (99%) rename go/test/endtoend/{backup_only => backup/vtbackup}/main_test.go (99%) diff --git a/go/test/endtoend/backup_only/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go similarity index 99% rename from go/test/endtoend/backup_only/backup_only_test.go rename to go/test/endtoend/backup/vtbackup/backup_only_test.go index ed7d325fb88..2dee35a7ba6 100644 --- a/go/test/endtoend/backup_only/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backup_only +package vtbackup import ( "bufio" diff --git a/go/test/endtoend/backup_only/main_test.go b/go/test/endtoend/backup/vtbackup/main_test.go similarity index 99% rename from go/test/endtoend/backup_only/main_test.go rename to go/test/endtoend/backup/vtbackup/main_test.go index e4dc4b83595..6a597371057 100644 --- a/go/test/endtoend/backup_only/main_test.go +++ b/go/test/endtoend/backup/vtbackup/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backup_only +package vtbackup import ( "flag" diff --git a/go/test/endtoend/cluster/vtbackup_process.go b/go/test/endtoend/cluster/vtbackup_process.go index b8f4d866545..01f93e67be3 100644 --- a/go/test/endtoend/cluster/vtbackup_process.go +++ b/go/test/endtoend/cluster/vtbackup_process.go @@ -122,8 +122,7 @@ func (vtbackup *VtbackupProcess) TearDown() error { // VtbackupProcessInstance returns a vtbackup handle // configured with the given Config. // The process must be manually started by calling Setup() - -func VtbackupProcessInstance(tabletUid int, mysqlPort int, newInitDBFile string, keyspace string, shard string, +func VtbackupProcessInstance(tabletUID int, mysqlPort int, newInitDBFile string, keyspace string, shard string, cell string, hostname string, tmpDirectory string, topoPort int, initialBackup bool) *VtbackupProcess { vtctl := VtctlProcessInstance(topoPort, hostname) vtbackup := &VtbackupProcess{ @@ -132,7 +131,7 @@ func VtbackupProcessInstance(tabletUid int, mysqlPort int, newInitDBFile string, CommonArg: *vtctl, LogDir: tmpDirectory, Directory: os.Getenv("VTDATAROOT"), - TabletAlias: fmt.Sprintf("%s-%010d", cell, tabletUid), + TabletAlias: fmt.Sprintf("%s-%010d", cell, tabletUID), initDBfile: newInitDBFile, Keyspace: keyspace, Shard: shard, From 366fe54ab5ea731a76d1efc0239e807b3854b174 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Fri, 10 Jan 2020 13:41:05 +0530 Subject: [PATCH 16/50] updating package name and fixing a test Signed-off-by: Ajeet jain --- .../backup/vtbackup/backup_only_test.go | 57 ++++++++++--------- .../backup/{ => vtctlbackup}/backup_test.go | 2 +- .../backup/{ => vtctlbackup}/main_test.go | 2 +- test/config.json | 20 +++---- 4 files changed, 43 insertions(+), 38 deletions(-) rename go/test/endtoend/backup/{ => vtctlbackup}/backup_test.go (99%) rename go/test/endtoend/backup/{ => vtctlbackup}/main_test.go (99%) diff --git a/go/test/endtoend/backup/vtbackup/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go index 2dee35a7ba6..b57907cbcb9 100644 --- a/go/test/endtoend/backup/vtbackup/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -70,6 +70,7 @@ func TestTabletInitialBackup(t *testing.T) { // Run the entire backup test firstBackupTest(t, "replica") + tearDown(t, true) } func TestTabletBackupOnly(t *testing.T) { // Test Backup Flow @@ -88,6 +89,8 @@ func TestTabletBackupOnly(t *testing.T) { initTablets(t, true, true) firstBackupTest(t, "replica") + + tearDown(t, false) } func firstBackupTest(t *testing.T, tabletType string) { @@ -155,7 +158,6 @@ func firstBackupTest(t *testing.T, tabletType string) { backups = countBackups(t) assert.Equal(t, 0, backups) - tearDown(t) } func vtBackup(t *testing.T, initialBackup bool) { @@ -243,7 +245,7 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS // Erase mysql/tablet dir, then start tablet with restore enabled. log.Info("restoring tablet %s", time.Now()) - resetTabletDirectory(t, *tablet) + resetTabletDirectory(t, *tablet, true) err := tablet.VttabletProcess.CreateDB(keyspaceName) assert.Nil(t, err) @@ -257,7 +259,7 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS assert.Nil(t, err) } -func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet) { +func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet, initMysql bool) { extraArgs := []string{"-db-credentials-file", dbCredentialFile} tablet.MysqlctlProcess.ExtraArgs = extraArgs @@ -273,36 +275,39 @@ func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet) { err = os.RemoveAll(tablet.VttabletProcess.Directory) assert.Nil(t, err) - // Init the Mysql - tablet.MysqlctlProcess.InitDBFile = newInitDBFile - err = tablet.MysqlctlProcess.Start() - assert.Nil(t, err) + if initMysql { + // Init the Mysql + tablet.MysqlctlProcess.InitDBFile = newInitDBFile + err = tablet.MysqlctlProcess.Start() + assert.Nil(t, err) + } + } -func tearDown(t *testing.T) { +func tearDown(t *testing.T, initMysql bool) { for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { //Tear down Tablet - err := tablet.VttabletProcess.TearDown() - assert.Nil(t, err) - err = localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) + //err := tablet.VttabletProcess.TearDown() + //assert.Nil(t, err) + err := localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) assert.Nil(t, err) - //resetTabletDirectory(t, tablet) + resetTabletDirectory(t, tablet, initMysql) } - // reset replication - promoteSlaveCommands := "STOP SLAVE; RESET SLAVE ALL; RESET MASTER;" - disableSemiSyncCommands := "SET GLOBAL rpl_semi_sync_master_enabled = false; SET GLOBAL rpl_semi_sync_slave_enabled = false" - for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { - _, err := tablet.VttabletProcess.QueryTablet(promoteSlaveCommands, keyspaceName, true) - assert.Nil(t, err) - _, err = tablet.VttabletProcess.QueryTablet(disableSemiSyncCommands, keyspaceName, true) - assert.Nil(t, err) - - for _, db := range []string{"_vt", "vt_insert_test"} { - _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("drop database if exists %s", db), keyspaceName, true) - assert.Nil(t, err) - } - } + //// reset replication + //promoteSlaveCommands := "STOP SLAVE; RESET SLAVE ALL; RESET MASTER;" + //disableSemiSyncCommands := "SET GLOBAL rpl_semi_sync_master_enabled = false; SET GLOBAL rpl_semi_sync_slave_enabled = false" + //for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { + // _, err := tablet.VttabletProcess.QueryTablet(promoteSlaveCommands, keyspaceName, true) + // assert.Nil(t, err) + // _, err = tablet.VttabletProcess.QueryTablet(disableSemiSyncCommands, keyspaceName, true) + // assert.Nil(t, err) + // + // for _, db := range []string{"_vt", "vt_insert_test"} { + // _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("drop database if exists %s", db), keyspaceName, true) + // assert.Nil(t, err) + // } + //} } diff --git a/go/test/endtoend/backup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go similarity index 99% rename from go/test/endtoend/backup/backup_test.go rename to go/test/endtoend/backup/vtctlbackup/backup_test.go index 162efb143c0..c6c83798102 100644 --- a/go/test/endtoend/backup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backup +package vtctlbackup import ( "bufio" diff --git a/go/test/endtoend/backup/main_test.go b/go/test/endtoend/backup/vtctlbackup/main_test.go similarity index 99% rename from go/test/endtoend/backup/main_test.go rename to go/test/endtoend/backup/vtctlbackup/main_test.go index 8c23f5a500c..aa95b77e951 100644 --- a/go/test/endtoend/backup/main_test.go +++ b/go/test/endtoend/backup/vtctlbackup/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backup +package vtctlbackup import ( "flag" diff --git a/test/config.json b/test/config.json index a61ce65eab0..af5f0ba4f7d 100644 --- a/test/config.json +++ b/test/config.json @@ -1,14 +1,5 @@ { "Tests": { - "backup_only": { - "File": "backup_only.py", - "Args": [], - "Command": [], - "Manual": false, - "Shard": 2, - "RetryMax": 0, - "Tags": [] - }, "backup_mysqlctld": { "File": "backup_mysqlctld.py", "Args": [], @@ -227,7 +218,16 @@ }, "backup": { "File": "backup.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backup"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtctlbackup"], + "Command": [], + "Manual": false, + "Shard": 11, + "RetryMax": 0, + "Tags": [] + }, + "backup_only": { + "File": "backup_only.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtbackup"], "Command": [], "Manual": false, "Shard": 11, From f887be382c17196ec9860f2a27b077036bc54d8e Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Mon, 13 Jan 2020 10:02:14 +0530 Subject: [PATCH 17/50] removed unrequired teardown code Signed-off-by: Ajeet jain --- go/test/endtoend/cluster/cluster_process.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index 680abbb1fa5..f3e75e845ff 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -484,10 +484,6 @@ func (cluster *LocalProcessCluster) Teardown() { log.Errorf("Error in etcd teardown - %s", err.Error()) } - if err := cluster.VtbackupProcess.TearDown(); err != nil { - log.Errorf("Error in VtBackup teardown - %s", err.Error()) - } - } // StartVtworker starts a vtworker From 29569e8027c0db60011d5e02bb6eb23654b02988 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Mon, 13 Jan 2020 10:13:21 +0530 Subject: [PATCH 18/50] Removed debug code Signed-off-by: Ajeet jain --- go/test/endtoend/backup/vtbackup/backup_only_test.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/go/test/endtoend/backup/vtbackup/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go index b57907cbcb9..0f987f16d21 100644 --- a/go/test/endtoend/backup/vtbackup/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -18,7 +18,6 @@ package vtbackup import ( "bufio" - "fmt" "os" "path" "strings" @@ -137,7 +136,7 @@ func firstBackupTest(t *testing.T, tabletType string) { err = localCluster.VtctlclientProcess.InitTablet(replica2, cell, keyspaceName, hostname, shardName) assert.Nil(t, err) restore(t, replica2, "replica", "SERVING") - // Replica2 takes time to server. Sleeping for 5 sec. + // Replica2 takes time to serve. Sleeping for 5 sec. time.Sleep(5 * time.Second) //check the new slave has the data cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) @@ -229,14 +228,6 @@ func initTablets(t *testing.T, startTablet bool, initShardMaster bool) { if initShardMaster { // choose master and start replication err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, master.TabletUID) - - if err != nil { - fmt.Println("*********______________****************") - fmt.Println(err) - fmt.Println(master.HTTPPort) - fmt.Println(replica1.HTTPPort) - time.Sleep(5 * time.Minute) - } assert.Nil(t, err) } } From 833cd08b47dda3f0239334dbd54249318f9e5bfe Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Mon, 20 Jan 2020 15:43:51 +0530 Subject: [PATCH 19/50] inital commit for xtrabackup Signed-off-by: Ajeet jain --- .../backup/vtctlbackup/backup_test.go | 420 +----------- .../backup/vtctlbackup/backup_utils.go | 613 ++++++++++++++++++ .../backup/xtrabackup/xtrabackup_test.go | 39 ++ 3 files changed, 662 insertions(+), 410 deletions(-) create mode 100644 go/test/endtoend/backup/vtctlbackup/backup_utils.go create mode 100644 go/test/endtoend/backup/xtrabackup/xtrabackup_test.go diff --git a/go/test/endtoend/backup/vtctlbackup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go index c6c83798102..b98435a30c2 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -16,418 +16,18 @@ limitations under the License. package vtctlbackup -import ( - "bufio" - "encoding/json" - "os" - "os/exec" - "path" - "strings" - "syscall" - "testing" - "time" +import "testing" - "vitess.io/vitess/go/vt/proto/topodata" - - "github.com/stretchr/testify/assert" - "vitess.io/vitess/go/test/endtoend/cluster" -) - -var ( - vtInsertTest = ` - create table vt_insert_test ( - id bigint auto_increment, - msg varchar(64), - primary key (id) - ) Engine=InnoDB` -) - -type restoreMethod func(t *testing.T, tablet *cluster.Vttablet) - -func TestReplicaBackup(t *testing.T) { - testBackup(t, "replica") -} - -func TestRdonlyBackup(t *testing.T) { - testBackup(t, "rdonly") -} - -//- create a shard with master and replica1 only -//- run InitShardMaster -//- insert some data -//- take a backup on master -//- insert more data on the master -//- bring up tablet_replica2 after the fact, let it restore the backup -//- check all data is right (before+after backup data) -//- list the backup, remove it -func TestMasterBackup(t *testing.T) { - verifyInitialReplication(t) - - output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("Backup", master.Alias) - assert.NotNil(t, err) - assert.Contains(t, output, "type MASTER cannot take backup. if you really need to do this, rerun the backup command with -allow_master") - - localCluster.VerifyBackupCount(t, shardKsName, 0) - - err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", "-allow_master=true", master.Alias) - assert.Nil(t, err) - - backups := localCluster.VerifyBackupCount(t, shardKsName, 1) - assert.Contains(t, backups[0], master.Alias) - - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - - restoreWaitForBackup(t, "replica") - err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) - assert.Nil(t, err) - - cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) - cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell) - verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups) - - replica2.VttabletProcess.TearDown() - master.VttabletProcess.QueryTablet("DROP TABLE vt_insert_test", keyspaceName, true) -} - -// Test a master and replica from the same backup. -// -// Check that a replica and master both restored from the same backup -// can replicate successfully. -func TestMasterReplicaSameBackup(t *testing.T) { - // insert data on master, wait for replica to get it - verifyInitialReplication(t) - - // backup the replica - err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - // insert more data on the master - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - - // now bring up the other replica, letting it restore from backup. - restoreWaitForBackup(t, "replica") - err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) - assert.Nil(t, err) - - // check the new replica has the data - cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) - - // Promote replica2 to master - err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", - "-keyspace_shard", shardKsName, - "-new_master", replica2.Alias) - assert.Nil(t, err) - - // insert more data on replica2 (current master) - _, err = replica2.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) - assert.Nil(t, err) - - // Force replica1 to restore from backup. - verifyRestoreTablet(t, replica1, "SERVING") - - // wait for replica1 to catch up. - cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 3) - - // This is to test that replicationPosition is processed correctly - // while doing backup/restore after a reparent. - // It is written into the MANIFEST and read back from the MANIFEST. - // - // Take another backup on the replica. - err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - // Insert more data on replica2 (current master). - _, err = replica2.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test4')", keyspaceName, true) - assert.Nil(t, err) - - // Force replica1 to restore from backup. - verifyRestoreTablet(t, replica1, "SERVING") - - cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 4) - replica2.VttabletProcess.TearDown() - restartMasterReplica(t) -} - -func TestRestoreOldMasterByRestart(t *testing.T) { - testRestoreOldMaster(t, restoreUsingRestart) -} - -func TestRestoreOldMasterInPlace(t *testing.T) { - testRestoreOldMaster(t, restoreInPlace) -} - -//Test that a former master replicates correctly after being restored. -// -//- Take a backup. -//- Reparent from old master to new master. -//- Force old master to restore from a previous backup using restore_method. -// -//Args: -//restore_method: function accepting one parameter of type tablet.Tablet, -//this function is called to force a restore on the provided tablet -// -func testRestoreOldMaster(t *testing.T, method restoreMethod) { - // insert data on master, wait for replica to get it - verifyInitialReplication(t) - - // backup the replica - err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - // insert more data on the master - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - - // reparent to replica1 - err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", - "-keyspace_shard", shardKsName, - "-new_master", replica1.Alias) - assert.Nil(t, err) - - // insert more data to new master - _, err = replica1.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) - assert.Nil(t, err) - - // force the old master to restore at the latest backup. - method(t, master) - - // wait for it to catch up. - cluster.VerifyRowsInTablet(t, master, keyspaceName, 3) - - // teardown - restartMasterReplica(t) -} - -func restoreUsingRestart(t *testing.T, tablet *cluster.Vttablet) { - tablet.VttabletProcess.TearDown() - verifyRestoreTablet(t, tablet, "SERVING") -} - -func restoreInPlace(t *testing.T, tablet *cluster.Vttablet) { - err := localCluster.VtctlclientProcess.ExecuteCommand("RestoreFromBackup", tablet.Alias) - assert.Nil(t, err) -} - -func restartMasterReplica(t *testing.T) { - // Stop all master, replica tablet and mysql instance - stopAllTablets() - - // remove all backups - localCluster.RemoveAllBackups(t, shardKsName) - - // start all tablet and mysql instances - var mysqlProcs []*exec.Cmd - for _, tablet := range []*cluster.Vttablet{master, replica1} { - proc, _ := tablet.MysqlctlProcess.StartProcess() - mysqlProcs = append(mysqlProcs, proc) - - err := localCluster.VtctlclientProcess.InitTablet(tablet, cell, keyspaceName, hostname, shardName) - assert.Nil(t, err) - tablet.VttabletProcess.CreateDB(keyspaceName) - tablet.VttabletProcess.Setup() - } - for _, proc := range mysqlProcs { - proc.Wait() - } - err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, master.TabletUID) - assert.Nil(t, err) -} - -func stopAllTablets() { - var mysqlProcs []*exec.Cmd - for _, tablet := range []*cluster.Vttablet{master, replica1, replica2} { - tablet.VttabletProcess.TearDown() - proc, _ := tablet.MysqlctlProcess.StopProcess() - mysqlProcs = append(mysqlProcs, proc) - localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) - } - for _, proc := range mysqlProcs { - proc.Wait() +// TestBackupMain - main tests backup using vtctl commands +func TestBackupMain(t *testing.T) { + code, err := LaunchCluster(false) + if err != nil { + t.Errorf("setup failed with status code %d", code) } - for _, tablet := range []*cluster.Vttablet{master, replica1} { - os.RemoveAll(tablet.VttabletProcess.Directory) - } -} - -func TestTerminatedRestore(t *testing.T) { - // insert data on master, wait for replica to get it - verifyInitialReplication(t) - - // backup the replica - err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - // insert more data on the master - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - - // reparent to replica1 - err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", - "-keyspace_shard", shardKsName, - "-new_master", replica1.Alias) - assert.Nil(t, err) - - // insert more data to new master - _, err = replica1.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) - assert.Nil(t, err) - - terminateRestore(t) - - err = localCluster.VtctlclientProcess.ExecuteCommand("RestoreFromBackup", master.Alias) - assert.Nil(t, err) - - output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", master.Alias) - assert.Nil(t, err) - - var tabletPB topodata.Tablet - err = json.Unmarshal([]byte(output), &tabletPB) - assert.Nil(t, err) - assert.Equal(t, tabletPB.Type, topodata.TabletType_REPLICA) - - _, err = os.Stat(path.Join(master.VttabletProcess.Directory, "restore_in_progress")) - assert.True(t, os.IsNotExist(err)) - - cluster.VerifyRowsInTablet(t, master, keyspaceName, 3) - stopAllTablets() -} - -//test_backup will: -//- create a shard with master and replica1 only -//- run InitShardMaster -//- bring up tablet_replica2 concurrently, telling it to wait for a backup -//- insert some data -//- take a backup -//- insert more data on the master -//- wait for tablet_replica2 to become SERVING -//- check all data is right (before+after backup data) -//- list the backup, remove it -// -//Args: -//tablet_type: 'replica' or 'rdonly'. -// -// -func testBackup(t *testing.T, tabletType string) { - restoreWaitForBackup(t, tabletType) - verifyInitialReplication(t) - - err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - backups := localCluster.VerifyBackupCount(t, shardKsName, 1) - - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) - assert.Nil(t, err) - cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + // Run all the backup tests + TestBackup(t) - cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell) - verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups) - - replica2.VttabletProcess.TearDown() - localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", replica2.Alias) - master.VttabletProcess.QueryTablet("DROP TABLE vt_insert_test", keyspaceName, true) - -} - -// This will create schema in master, insert some data to master and verify the same data in replica -func verifyInitialReplication(t *testing.T) { - _, err := master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true) - assert.Nil(t, err) - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test1')", keyspaceName, true) - assert.Nil(t, err) - cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 1) -} - -// Bring up another replica concurrently, telling it to wait until a backup -// is available instead of starting up empty. -// -// Override the backup engine implementation to a non-existent one for restore. -// This setting should only matter for taking new backups. We should be able -// to restore a previous backup successfully regardless of this setting. -func restoreWaitForBackup(t *testing.T, tabletType string) { - replica2.Type = tabletType - resetTabletDir(t, replica2) - replicaTabletArgs := commonTabletArg - replicaTabletArgs = append(replicaTabletArgs, "-backup_engine_implementation", "fake_implementation") - replicaTabletArgs = append(replicaTabletArgs, "-wait_for_backup_interval", "1s") - replicaTabletArgs = append(replicaTabletArgs, "-init_tablet_type", tabletType) - replica2.VttabletProcess.ExtraArgs = replicaTabletArgs - replica2.VttabletProcess.ServingStatus = "" - err := replica2.VttabletProcess.Setup() - assert.Nil(t, err) -} - -func resetTabletDir(t *testing.T, tablet *cluster.Vttablet) { - err := cluster.ResetTabletDirectory(*tablet) - assert.Nil(t, err) -} - -func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []string) { - // Remove the backup - for _, backup := range backups { - err := localCluster.VtctlclientProcess.ExecuteCommand("RemoveBackup", shardKsName, backup) - assert.Nil(t, err) - } - - // Now, there should not be no backup - localCluster.VerifyBackupCount(t, shardKsName, 0) -} - -func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string) { - tablet.VttabletProcess.TearDown() - resetTabletDir(t, tablet) - tablet.VttabletProcess.ServingStatus = "" - err := tablet.VttabletProcess.Setup() - assert.Nil(t, err) - if status != "" { - err = tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{status}, 25*time.Second) - assert.Nil(t, err) - } - - if tablet.Type == "replica" { - verifyReplicationStatus(t, tablet, "ON") - } else if tablet.Type == "rdonly" { - verifyReplicationStatus(t, tablet, "OFF") - } -} - -func verifyReplicationStatus(t *testing.T, vttablet *cluster.Vttablet, expectedStatus string) { - status, err := vttablet.VttabletProcess.GetDBVar("rpl_semi_sync_slave_enabled", keyspaceName) - assert.Nil(t, err) - assert.Equal(t, status, expectedStatus) - status, err = vttablet.VttabletProcess.GetDBStatus("rpl_semi_sync_slave_status", keyspaceName) - assert.Nil(t, err) - assert.Equal(t, status, expectedStatus) -} - -func terminateRestore(t *testing.T) { - stopRestoreMsg := "Copying file 10" - args := append([]string{"-server", localCluster.VtctlclientProcess.Server, "-alsologtostderr"}, "RestoreFromBackup", master.Alias) - tmpProcess := exec.Command( - "vtctlclient", - args..., - ) - - reader, _ := tmpProcess.StderrPipe() - err := tmpProcess.Start() - assert.Nil(t, err) - found := false - - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - text := scanner.Text() - if strings.Contains(text, stopRestoreMsg) { - if _, err := os.Stat(path.Join(master.VttabletProcess.Directory, "restore_in_progress")); os.IsNotExist(err) { - assert.Fail(t, "restore in progress file missing") - } - tmpProcess.Process.Signal(syscall.SIGTERM) - found = true - return - } - } - assert.True(t, found, "Restore message not found") + // Teardown the cluster + TearDownCluster() } diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go new file mode 100644 index 00000000000..2164e635b2c --- /dev/null +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -0,0 +1,613 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctlbackup + +import ( + "bufio" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + "syscall" + "testing" + "time" + + "vitess.io/vitess/go/test/endtoend/sharding/initialsharding" + + "vitess.io/vitess/go/vt/proto/topodata" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + master *cluster.Vttablet + replica1 *cluster.Vttablet + replica2 *cluster.Vttablet + localCluster *cluster.LocalProcessCluster + newInitDBFile string + cell = cluster.DefaultCell + hostname = "localhost" + keyspaceName = "ks" + dbPassword = "VtDbaPass" + shardKsName = fmt.Sprintf("%s/%s", keyspaceName, shardName) + dbCredentialFile string + shardName = "0" + commonTabletArg = []string{ + "-vreplication_healthcheck_topology_refresh", "1s", + "-vreplication_healthcheck_retry_delay", "1s", + "-vreplication_retry_delay", "1s", + "-degraded_threshold", "5s", + "-lock_tables_timeout", "5s", + "-watch_replication_stream", + "-enable_replication_reporter", + "-serving_state_grace_period", "1s"} + xtrabackupArgs = []string{ + "-backup_engine_implementation", + "xtrabackup", + "-xtrabackup_stream_mode", "tar", + "-xtrabackup_user=vt_dba", + "-xtrabackup_stripes=0", + "-xtrabackup_backup_flags", + fmt.Sprintf("--password=%s", dbPassword)} + + vtInsertTest = ` + create table vt_insert_test ( + id bigint auto_increment, + msg varchar(64), + primary key (id) + ) Engine=InnoDB` +) + +func LaunchCluster(xtrabackup bool) (int, error) { + localCluster = cluster.NewCluster(cell, hostname) + + // Start topo server + err := localCluster.StartTopo() + if err != nil { + return 1, err + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + } + localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace) + + dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory) + initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) + sql := string(initDb) + newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") + sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + err = ioutil.WriteFile(newInitDBFile, []byte(sql), 0666) + if err != nil { + return 1, err + } + + extraArgs := []string{"-db-credentials-file", dbCredentialFile} + commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) + + if xtrabackup { + commonTabletArg = append(commonTabletArg, xtrabackupArgs...) + } + + shard := cluster.Shard{ + Name: shardName, + } + + var mysqlProcs []*exec.Cmd + for i := 0; i < 3; i++ { + tabletType := "replica" + if i == 0 { + tabletType = "master" + } + tablet := localCluster.GetVttabletInstance(tabletType, 0, cell) + tablet.VttabletProcess = localCluster.GetVtprocessInstanceFromVttablet(tablet, shard.Name, keyspaceName) + tablet.VttabletProcess.DbPassword = dbPassword + tablet.VttabletProcess.ExtraArgs = commonTabletArg + tablet.VttabletProcess.SupportsBackup = true + tablet.VttabletProcess.EnableSemiSync = true + + tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) + tablet.MysqlctlProcess.InitDBFile = newInitDBFile + tablet.MysqlctlProcess.ExtraArgs = extraArgs + if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { + return 1, err + } else { + mysqlProcs = append(mysqlProcs, proc) + } + shard.Vttablets = append(shard.Vttablets, tablet) + } + for _, proc := range mysqlProcs { + if err := proc.Wait(); err != nil { + return 1, err + } + } + master = shard.Vttablets[0] + replica1 = shard.Vttablets[1] + replica2 = shard.Vttablets[2] + + if err := localCluster.VtctlclientProcess.InitTablet(master, cell, keyspaceName, hostname, shard.Name); err != nil { + return 1, err + } + if err := localCluster.VtctlclientProcess.InitTablet(replica1, cell, keyspaceName, hostname, shard.Name); err != nil { + return 1, err + } + + for _, tablet := range []cluster.Vttablet{*master, *replica1} { + if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil { + return 1, err + } + if err := tablet.VttabletProcess.Setup(); err != nil { + return 1, err + } + } + + if err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shard.Name, cell, master.TabletUID); err != nil { + return 1, err + } + return 0, nil +} + +func TearDownCluster() { + localCluster.Teardown() +} + +func TestBackup(t *testing.T) { + // Run all the backup tests + t.Run("TestReplicaBackup", func(t *testing.T) { + vtctlBackup(t, "replica") + }) + + t.Run("TestRdonlyBackup", func(t *testing.T) { + vtctlBackup(t, "rdonly") + }) + + t.Run("TestMasterBackup", func(t *testing.T) { + masterBackup(t) + }) + + t.Run("TestMasterReplicaSameBackup", func(t *testing.T) { + masterReplicaSameBackup(t) + }) + + t.Run("TestRestoreOldMasterByRestart", func(t *testing.T) { + restoreOldMasterByRestart(t) + }) + + t.Run("TestRestoreOldMasterInPlace", func(t *testing.T) { + restoreOldMasterInPlace(t) + }) + + t.Run("TestTerminatedRestore", func(t *testing.T) { + terminatedRestore(t) + }) + +} + +type restoreMethod func(t *testing.T, tablet *cluster.Vttablet) + +//- create a shard with master and replica1 only +//- run InitShardMaster +//- insert some data +//- take a backup on master +//- insert more data on the master +//- bring up tablet_replica2 after the fact, let it restore the backup +//- check all data is right (before+after backup data) +//- list the backup, remove it +func masterBackup(t *testing.T) { + verifyInitialReplication(t) + + output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("Backup", master.Alias) + assert.NotNil(t, err) + assert.Contains(t, output, "type MASTER cannot take backup. if you really need to do this, rerun the backup command with -allow_master") + + backups := listBackups(t) + assert.Equal(t, len(backups), 0) + + err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", "-allow_master=true", master.Alias) + assert.Nil(t, err) + + backups = listBackups(t) + assert.Equal(t, len(backups), 1) + assert.Contains(t, backups[0], master.Alias) + + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + + restoreWaitForBackup(t, "replica") + err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) + assert.Nil(t, err) + + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell) + verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups) + + err = replica2.VttabletProcess.TearDown() + assert.Nil(t, err) + + _, err = master.VttabletProcess.QueryTablet("DROP TABLE vt_insert_test", keyspaceName, true) + assert.Nil(t, err) +} + +// Test a master and replica from the same backup. +// +// Check that a replica and master both restored from the same backup +// can replicate successfully. +func masterReplicaSameBackup(t *testing.T) { + // insert data on master, wait for replica to get it + verifyInitialReplication(t) + + // backup the replica + err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + + // insert more data on the master + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + + // now bring up the other replica, letting it restore from backup. + restoreWaitForBackup(t, "replica") + err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) + assert.Nil(t, err) + + // check the new replica has the data + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + + // Promote replica2 to master + err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", + "-keyspace_shard", shardKsName, + "-new_master", replica2.Alias) + assert.Nil(t, err) + + // insert more data on replica2 (current master) + _, err = replica2.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) + assert.Nil(t, err) + + // Force replica1 to restore from backup. + verifyRestoreTablet(t, replica1, "SERVING") + + // wait for replica1 to catch up. + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 3) + + // This is to test that replicationPosition is processed correctly + // while doing backup/restore after a reparent. + // It is written into the MANIFEST and read back from the MANIFEST. + // + // Take another backup on the replica. + err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + + // Insert more data on replica2 (current master). + _, err = replica2.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test4')", keyspaceName, true) + assert.Nil(t, err) + + // Force replica1 to restore from backup. + verifyRestoreTablet(t, replica1, "SERVING") + + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 4) + err = replica2.VttabletProcess.TearDown() + assert.Nil(t, err) + restartMasterReplica(t) +} + +func restoreOldMasterByRestart(t *testing.T) { + testRestoreOldMaster(t, restoreUsingRestart) +} + +func restoreOldMasterInPlace(t *testing.T) { + testRestoreOldMaster(t, restoreInPlace) +} + +//Test that a former master replicates correctly after being restored. +// +//- Take a backup. +//- Reparent from old master to new master. +//- Force old master to restore from a previous backup using restore_method. +// +//Args: +//restore_method: function accepting one parameter of type tablet.Tablet, +//this function is called to force a restore on the provided tablet +// +func testRestoreOldMaster(t *testing.T, method restoreMethod) { + // insert data on master, wait for replica to get it + verifyInitialReplication(t) + + // backup the replica + err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + + // insert more data on the master + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + + // reparent to replica1 + err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", + "-keyspace_shard", shardKsName, + "-new_master", replica1.Alias) + assert.Nil(t, err) + + // insert more data to new master + _, err = replica1.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) + assert.Nil(t, err) + + // force the old master to restore at the latest backup. + method(t, master) + + // wait for it to catch up. + cluster.VerifyRowsInTablet(t, master, keyspaceName, 3) + + // teardown + restartMasterReplica(t) +} + +func restoreUsingRestart(t *testing.T, tablet *cluster.Vttablet) { + err := tablet.VttabletProcess.TearDown() + assert.Nil(t, err) + verifyRestoreTablet(t, tablet, "SERVING") +} + +func restoreInPlace(t *testing.T, tablet *cluster.Vttablet) { + err := localCluster.VtctlclientProcess.ExecuteCommand("RestoreFromBackup", tablet.Alias) + assert.Nil(t, err) +} + +func restartMasterReplica(t *testing.T) { + // Stop all master, replica tablet and mysql instance + stopAllTablets() + + // remove all backups + backups := listBackups(t) + for _, backup := range backups { + err := localCluster.VtctlclientProcess.ExecuteCommand("RemoveBackup", shardKsName, backup) + assert.Nil(t, err) + } + // start all tablet and mysql instances + var mysqlProcs []*exec.Cmd + for _, tablet := range []*cluster.Vttablet{master, replica1} { + proc, _ := tablet.MysqlctlProcess.StartProcess() + mysqlProcs = append(mysqlProcs, proc) + + err := localCluster.VtctlclientProcess.InitTablet(tablet, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + err = tablet.VttabletProcess.CreateDB(keyspaceName) + assert.Nil(t, err) + err = tablet.VttabletProcess.Setup() + assert.Nil(t, err) + } + for _, proc := range mysqlProcs { + proc.Wait() + } + err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, master.TabletUID) + assert.Nil(t, err) +} + +func stopAllTablets() { + var mysqlProcs []*exec.Cmd + for _, tablet := range []*cluster.Vttablet{master, replica1, replica2} { + tablet.VttabletProcess.TearDown() + proc, _ := tablet.MysqlctlProcess.StopProcess() + mysqlProcs = append(mysqlProcs, proc) + localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) + } + for _, proc := range mysqlProcs { + proc.Wait() + } + for _, tablet := range []*cluster.Vttablet{master, replica1} { + os.RemoveAll(tablet.VttabletProcess.Directory) + } +} + +func terminatedRestore(t *testing.T) { + // insert data on master, wait for replica to get it + verifyInitialReplication(t) + + // backup the replica + err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + + // insert more data on the master + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + + // reparent to replica1 + err = localCluster.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", + "-keyspace_shard", shardKsName, + "-new_master", replica1.Alias) + assert.Nil(t, err) + + // insert more data to new master + _, err = replica1.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test3')", keyspaceName, true) + assert.Nil(t, err) + + terminateRestore(t) + + err = localCluster.VtctlclientProcess.ExecuteCommand("RestoreFromBackup", master.Alias) + assert.Nil(t, err) + + output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", master.Alias) + assert.Nil(t, err) + + var tabletPB topodata.Tablet + err = json.Unmarshal([]byte(output), &tabletPB) + assert.Nil(t, err) + assert.Equal(t, tabletPB.Type, topodata.TabletType_REPLICA) + + _, err = os.Stat(path.Join(master.VttabletProcess.Directory, "restore_in_progress")) + assert.True(t, os.IsNotExist(err)) + + cluster.VerifyRowsInTablet(t, master, keyspaceName, 3) + stopAllTablets() +} + +//test_backup will: +//- create a shard with master and replica1 only +//- run InitShardMaster +//- bring up tablet_replica2 concurrently, telling it to wait for a backup +//- insert some data +//- take a backup +//- insert more data on the master +//- wait for tablet_replica2 to become SERVING +//- check all data is right (before+after backup data) +//- list the backup, remove it +// +//Args: +//tablet_type: 'replica' or 'rdonly'. +// +// +func vtctlBackup(t *testing.T, tabletType string) { + restoreWaitForBackup(t, tabletType) + verifyInitialReplication(t) + + err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + fmt.Println(err) + + backups := listBackups(t) + assert.Equal(t, len(backups), 1) + time.Sleep(10 * time.Second) + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + fmt.Println(err) + + err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) + assert.Nil(t, err) + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + + cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell) + verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups) + + err = replica2.VttabletProcess.TearDown() + assert.Nil(t, err) + + err = localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", replica2.Alias) + assert.Nil(t, err) + _, err = master.VttabletProcess.QueryTablet("DROP TABLE vt_insert_test", keyspaceName, true) + assert.Nil(t, err) + +} + +// This will create schema in master, insert some data to master and verify the same data in replica +func verifyInitialReplication(t *testing.T) { + _, err := master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true) + assert.Nil(t, err) + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test1')", keyspaceName, true) + assert.Nil(t, err) + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 1) +} + +// Bring up another replica concurrently, telling it to wait until a backup +// is available instead of starting up empty. +// +// Override the backup engine implementation to a non-existent one for restore. +// This setting should only matter for taking new backups. We should be able +// to restore a previous backup successfully regardless of this setting. +func restoreWaitForBackup(t *testing.T, tabletType string) { + replica2.Type = tabletType + resetTabletDir(t, replica2) + replicaTabletArgs := commonTabletArg + replicaTabletArgs = append(replicaTabletArgs, "-backup_engine_implementation", "fake_implementation") + replicaTabletArgs = append(replicaTabletArgs, "-wait_for_backup_interval", "1s") + replicaTabletArgs = append(replicaTabletArgs, "-init_tablet_type", tabletType) + replica2.VttabletProcess.ExtraArgs = replicaTabletArgs + replica2.VttabletProcess.ServingStatus = "" + err := replica2.VttabletProcess.Setup() + assert.Nil(t, err) +} + +func resetTabletDir(t *testing.T, tablet *cluster.Vttablet) { + err := cluster.ResetTabletDirectory(*tablet) + assert.Nil(t, err) +} + +func listBackups(t *testing.T) []string { + output, err := localCluster.ListBackups(shardKsName) + assert.Nil(t, err) + return output +} + +func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []string) { + // Remove the backup + for _, backup := range backups { + err := localCluster.VtctlclientProcess.ExecuteCommand("RemoveBackup", shardKsName, backup) + assert.Nil(t, err) + } + + // Now, there should not be no backup + backups = listBackups(t) + assert.Equal(t, len(backups), 0) +} + +func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string) { + err := tablet.VttabletProcess.TearDown() + assert.Nil(t, err) + + resetTabletDir(t, tablet) + tablet.VttabletProcess.ServingStatus = "" + err = tablet.VttabletProcess.Setup() + assert.Nil(t, err) + if status != "" { + err = tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{status}, 25*time.Second) + assert.Nil(t, err) + } + + if tablet.Type == "replica" { + verifyReplicationStatus(t, tablet, "ON") + } else if tablet.Type == "rdonly" { + verifyReplicationStatus(t, tablet, "OFF") + } +} + +func verifyReplicationStatus(t *testing.T, vttablet *cluster.Vttablet, expectedStatus string) { + status, err := vttablet.VttabletProcess.GetDBVar("rpl_semi_sync_slave_enabled", keyspaceName) + assert.Nil(t, err) + assert.Equal(t, status, expectedStatus) + status, err = vttablet.VttabletProcess.GetDBStatus("rpl_semi_sync_slave_status", keyspaceName) + assert.Nil(t, err) + assert.Equal(t, status, expectedStatus) +} + +func terminateRestore(t *testing.T) { + stopRestoreMsg := "Copying file 10" + args := append([]string{"-server", localCluster.VtctlclientProcess.Server, "-alsologtostderr"}, "RestoreFromBackup", master.Alias) + tmpProcess := exec.Command( + "vtctlclient", + args..., + ) + + reader, _ := tmpProcess.StderrPipe() + err := tmpProcess.Start() + assert.Nil(t, err) + found := false + + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + text := scanner.Text() + if strings.Contains(text, stopRestoreMsg) { + if _, err := os.Stat(path.Join(master.VttabletProcess.Directory, "restore_in_progress")); os.IsNotExist(err) { + assert.Fail(t, "restore in progress file missing") + } + tmpProcess.Process.Signal(syscall.SIGTERM) + found = true + return + } + } + assert.True(t, found, "Restore message not found") +} diff --git a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go new file mode 100644 index 00000000000..99ab22fbf53 --- /dev/null +++ b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go @@ -0,0 +1,39 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctlbackup + +import ( + "fmt" + "testing" + + backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" +) + +// TestXtraBackup - tests the back using xtrabackup arguments +func TestXtraBackup(t *testing.T) { + code, err := backup.LaunchCluster(true) + fmt.Println(err) + if err != nil { + t.Errorf("setup failed with status code %d", code) + } + + // Run all the backup tests + backup.TestBackup(t) + + // Teardown the cluster + backup.TearDownCluster() +} From 7e9f8378d21d38d821c128a5ef5870fe5046fc10 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 13:27:49 +0530 Subject: [PATCH 20/50] fix sequencing of cleanup Signed-off-by: Ajeet jain --- .../backup/vtctlbackup/backup_utils.go | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 2164e635b2c..6a5a03175f6 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -58,16 +58,15 @@ var ( "-lock_tables_timeout", "5s", "-watch_replication_stream", "-enable_replication_reporter", - "-serving_state_grace_period", "1s"} + "-serving_state_grace_period", "1s", + } xtrabackupArgs = []string{ - "-backup_engine_implementation", - "xtrabackup", + "-backup_engine_implementation", "xtrabackup", "-xtrabackup_stream_mode", "tar", "-xtrabackup_user=vt_dba", "-xtrabackup_stripes=0", - "-xtrabackup_backup_flags", - fmt.Sprintf("--password=%s", dbPassword)} - + "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), + } vtInsertTest = ` create table vt_insert_test ( id bigint auto_increment, @@ -384,7 +383,11 @@ func restartMasterReplica(t *testing.T) { for _, tablet := range []*cluster.Vttablet{master, replica1} { proc, _ := tablet.MysqlctlProcess.StartProcess() mysqlProcs = append(mysqlProcs, proc) - + } + for _, proc := range mysqlProcs { + proc.Wait() + } + for _, tablet := range []*cluster.Vttablet{master, replica1} { err := localCluster.VtctlclientProcess.InitTablet(tablet, cell, keyspaceName, hostname, shardName) assert.Nil(t, err) err = tablet.VttabletProcess.CreateDB(keyspaceName) @@ -392,9 +395,6 @@ func restartMasterReplica(t *testing.T) { err = tablet.VttabletProcess.Setup() assert.Nil(t, err) } - for _, proc := range mysqlProcs { - proc.Wait() - } err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, master.TabletUID) assert.Nil(t, err) } @@ -478,14 +478,12 @@ func vtctlBackup(t *testing.T, tabletType string) { err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) assert.Nil(t, err) - fmt.Println(err) backups := listBackups(t) assert.Equal(t, len(backups), 1) - time.Sleep(10 * time.Second) + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) assert.Nil(t, err) - fmt.Println(err) err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) assert.Nil(t, err) From 815b693c9312f8d7f4c2d9e6fbafe63ed9c0d41d Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 13:42:46 +0530 Subject: [PATCH 21/50] fix terminate restore for xtrabackup Signed-off-by: Ajeet jain --- go/test/endtoend/backup/vtctlbackup/backup_utils.go | 7 +++++++ go/test/endtoend/backup/xtrabackup/xtrabackup_test.go | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 6a5a03175f6..584563e3819 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -43,6 +43,7 @@ var ( replica2 *cluster.Vttablet localCluster *cluster.LocalProcessCluster newInitDBFile string + useXtrabackup bool cell = cluster.DefaultCell hostname = "localhost" keyspaceName = "ks" @@ -104,6 +105,7 @@ func LaunchCluster(xtrabackup bool) (int, error) { commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) if xtrabackup { + useXtrabackup = xtrabackup commonTabletArg = append(commonTabletArg, xtrabackupArgs...) } @@ -584,6 +586,10 @@ func verifyReplicationStatus(t *testing.T, vttablet *cluster.Vttablet, expectedS func terminateRestore(t *testing.T) { stopRestoreMsg := "Copying file 10" + if useXtrabackup { + stopRestoreMsg = "Restore: Preparing" + } + args := append([]string{"-server", localCluster.VtctlclientProcess.Server, "-alsologtostderr"}, "RestoreFromBackup", master.Alias) tmpProcess := exec.Command( "vtctlclient", @@ -596,6 +602,7 @@ func terminateRestore(t *testing.T) { found := false scanner := bufio.NewScanner(reader) + for scanner.Scan() { text := scanner.Text() if strings.Contains(text, stopRestoreMsg) { diff --git a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go index 99ab22fbf53..16244a01a48 100644 --- a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go +++ b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go @@ -17,7 +17,6 @@ limitations under the License. package vtctlbackup import ( - "fmt" "testing" backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" @@ -26,7 +25,6 @@ import ( // TestXtraBackup - tests the back using xtrabackup arguments func TestXtraBackup(t *testing.T) { code, err := backup.LaunchCluster(true) - fmt.Println(err) if err != nil { t.Errorf("setup failed with status code %d", code) } From 61fe132e4228110f0ac6ed982fbccf160c1f6dd7 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 15:02:57 +0530 Subject: [PATCH 22/50] updated config for xtrabackup Signed-off-by: Ajeet jain --- .github/workflows/cluster_endtoend.yml | 6 +++++- test/config.json | 18 +++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index 93fb133d8bd..399cf902ede 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - name: [11, 12, 13, 14, 15, 16, 17] + name: [11, 12, 13, 14, 15, 16, 17, 19] steps: - name: Set up Go @@ -26,6 +26,10 @@ jobs: sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 - name: sharded cluster_endtoend run: | diff --git a/test/config.json b/test/config.json index af5f0ba4f7d..fe4f3e121f0 100644 --- a/test/config.json +++ b/test/config.json @@ -189,15 +189,6 @@ "worker_test" ] }, - "xtrabackup": { - "File": "xtrabackup.py", - "Args": [], - "Command": [], - "Manual": false, - "Shard": 1, - "RetryMax": 0, - "Tags": [] - }, "xb_recovery": { "File": "xb_recovery.py", "Args": [], @@ -234,6 +225,15 @@ "RetryMax": 0, "Tags": [] }, + "backup_xtrabackup": { + "File": "xtrabackup.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backup/xtrabackup"], + "Command": [], + "Manual": false, + "Shard": 19, + "RetryMax": 0, + "Tags": [] + }, "binlog": { "File": "binlog.go", "Args": ["vitess.io/vitess/go/test/endtoend/binlog"], From a334d243e060b12bb122ab6959c9f85c5664a11f Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 15:45:34 +0530 Subject: [PATCH 23/50] added xtrabackup stream mode Signed-off-by: Ajeet jain --- .../backup/vtctlbackup/backup_test.go | 2 +- .../backup/vtctlbackup/backup_utils.go | 10 +++-- .../backup/xtrabackup/xtrabackup_test.go | 2 +- .../xtrabackup_stream_test.go | 37 +++++++++++++++++++ 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go diff --git a/go/test/endtoend/backup/vtctlbackup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go index b98435a30c2..f00f793b47d 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -20,7 +20,7 @@ import "testing" // TestBackupMain - main tests backup using vtctl commands func TestBackupMain(t *testing.T) { - code, err := LaunchCluster(false) + code, err := LaunchCluster(false, "", 0) if err != nil { t.Errorf("setup failed with status code %d", code) } diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 584563e3819..40f8e6aa592 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -44,6 +44,8 @@ var ( localCluster *cluster.LocalProcessCluster newInitDBFile string useXtrabackup bool + xbStreamMode string + xbStripes int cell = cluster.DefaultCell hostname = "localhost" keyspaceName = "ks" @@ -63,9 +65,9 @@ var ( } xtrabackupArgs = []string{ "-backup_engine_implementation", "xtrabackup", - "-xtrabackup_stream_mode", "tar", + "-xtrabackup_stream_mode", xbStreamMode, "-xtrabackup_user=vt_dba", - "-xtrabackup_stripes=0", + fmt.Sprintf("-xtrabackup_stripes=%d", xbStripes), "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), } vtInsertTest = ` @@ -76,7 +78,7 @@ var ( ) Engine=InnoDB` ) -func LaunchCluster(xtrabackup bool) (int, error) { +func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error) { localCluster = cluster.NewCluster(cell, hostname) // Start topo server @@ -106,6 +108,8 @@ func LaunchCluster(xtrabackup bool) (int, error) { if xtrabackup { useXtrabackup = xtrabackup + xbStreamMode = streamMode + xbStripes = stripes commonTabletArg = append(commonTabletArg, xtrabackupArgs...) } diff --git a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go index 16244a01a48..c6346bcb428 100644 --- a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go +++ b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go @@ -24,7 +24,7 @@ import ( // TestXtraBackup - tests the back using xtrabackup arguments func TestXtraBackup(t *testing.T) { - code, err := backup.LaunchCluster(true) + code, err := backup.LaunchCluster(true, "tar", 0) if err != nil { t.Errorf("setup failed with status code %d", code) } diff --git a/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go b/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go new file mode 100644 index 00000000000..e25b5e758bc --- /dev/null +++ b/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go @@ -0,0 +1,37 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctlbackup + +import ( + "testing" + + backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" +) + +// TestXtraBackup - tests the back using xtrabackup arguments +func TestXtraBackup(t *testing.T) { + code, err := backup.LaunchCluster(true, "xbstream", 8) + if err != nil { + t.Errorf("setup failed with status code %d", code) + } + + // Run all the backup tests + backup.TestBackup(t) + + // Teardown the cluster + backup.TearDownCluster() +} From d68b6c444006bf385c4cc6db858cace7d838d03c Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 15:51:02 +0530 Subject: [PATCH 24/50] updated config for xtrabackup stream Signed-off-by: Ajeet jain --- .github/workflows/cluster_endtoend.yml | 2 +- .../backup/vtctlbackup/backup_utils.go | 2 +- test/config.json | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index 399cf902ede..66f3b6bc2e0 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - name: [11, 12, 13, 14, 15, 16, 17, 19] + name: [11, 12, 13, 14, 15, 16, 17, 19, 20] steps: - name: Set up Go diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 40f8e6aa592..f6b121454d4 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -65,7 +65,7 @@ var ( } xtrabackupArgs = []string{ "-backup_engine_implementation", "xtrabackup", - "-xtrabackup_stream_mode", xbStreamMode, + fmt.Sprintf("-xtrabackup_stream_mode=%s", xbStreamMode), "-xtrabackup_user=vt_dba", fmt.Sprintf("-xtrabackup_stripes=%d", xbStripes), "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), diff --git a/test/config.json b/test/config.json index fe4f3e121f0..8f58db57851 100644 --- a/test/config.json +++ b/test/config.json @@ -198,15 +198,6 @@ "RetryMax": 0, "Tags": [] }, - "xtrabackup_xbstream": { - "File": "xtrabackup_xbstream.py", - "Args": [], - "Command": [], - "Manual": false, - "Shard": 3, - "RetryMax": 0, - "Tags": [] - }, "backup": { "File": "backup.go", "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtctlbackup"], @@ -234,6 +225,15 @@ "RetryMax": 0, "Tags": [] }, + "backup_xtrabackup_xbstream": { + "File": "xtrabackup_xbstream.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backup/xtrabackupstream"], + "Command": [], + "Manual": false, + "Shard": 20, + "RetryMax": 0, + "Tags": [] + }, "binlog": { "File": "binlog.go", "Args": ["vitess.io/vitess/go/test/endtoend/binlog"], From e10dbdb1bfc55d16571cab8f6ae15f299eecd321 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Tue, 21 Jan 2020 15:55:42 +0530 Subject: [PATCH 25/50] minor changes to config Signed-off-by: Ajeet jain --- .../backup/vtctlbackup/backup_utils.go | 18 +++++++++++------- .../backup/xtrabackup/xtrabackup_test.go | 4 ++-- .../xtrabackupstream/xtrabackup_stream_test.go | 4 ++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index f6b121454d4..33c955467ab 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -63,13 +63,7 @@ var ( "-enable_replication_reporter", "-serving_state_grace_period", "1s", } - xtrabackupArgs = []string{ - "-backup_engine_implementation", "xtrabackup", - fmt.Sprintf("-xtrabackup_stream_mode=%s", xbStreamMode), - "-xtrabackup_user=vt_dba", - fmt.Sprintf("-xtrabackup_stripes=%d", xbStripes), - "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), - } + vtInsertTest = ` create table vt_insert_test ( id bigint auto_increment, @@ -106,10 +100,20 @@ func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error) extraArgs := []string{"-db-credentials-file", dbCredentialFile} commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) + // Update arguments for xtrabackup if xtrabackup { useXtrabackup = xtrabackup xbStreamMode = streamMode xbStripes = stripes + + xtrabackupArgs := []string{ + "-backup_engine_implementation", "xtrabackup", + fmt.Sprintf("-xtrabackup_stream_mode=%s", xbStreamMode), + "-xtrabackup_user=vt_dba", + fmt.Sprintf("-xtrabackup_stripes=%d", xbStripes), + "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), + } + commonTabletArg = append(commonTabletArg, xtrabackupArgs...) } diff --git a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go index c6346bcb428..4b94086bcd3 100644 --- a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go +++ b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go @@ -22,8 +22,8 @@ import ( backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" ) -// TestXtraBackup - tests the back using xtrabackup arguments -func TestXtraBackup(t *testing.T) { +// TestXtraBackup - tests the backup using xtrabackup +func TestXtrabackup(t *testing.T) { code, err := backup.LaunchCluster(true, "tar", 0) if err != nil { t.Errorf("setup failed with status code %d", code) diff --git a/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go b/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go index e25b5e758bc..5e86e73719e 100644 --- a/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go +++ b/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go @@ -22,8 +22,8 @@ import ( backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" ) -// TestXtraBackup - tests the back using xtrabackup arguments -func TestXtraBackup(t *testing.T) { +// TestXtrabackupStream - tests the backup using xtrabackup with xbstream mode +func TestXtrabackupStream(t *testing.T) { code, err := backup.LaunchCluster(true, "xbstream", 8) if err != nil { t.Errorf("setup failed with status code %d", code) From c1e8706f5c3d78fc2165d02e58cd707d3f402884 Mon Sep 17 00:00:00 2001 From: Arindam Nayak Date: Wed, 22 Jan 2020 11:52:08 +0530 Subject: [PATCH 26/50] rebased to resolve conflict Signed-off-by: Arindam Nayak --- .../backup/vtctlbackup/backup_utils.go | 27 +-- .../endtoend/backup/vtctlbackup/main_test.go | 158 ------------------ 2 files changed, 8 insertions(+), 177 deletions(-) delete mode 100644 go/test/endtoend/backup/vtctlbackup/main_test.go diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 33c955467ab..f6a0c87b0b7 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -72,6 +72,7 @@ var ( ) Engine=InnoDB` ) +// LaunchCluster builds cluster with required setup to test backup func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error) { localCluster = cluster.NewCluster(cell, hostname) @@ -175,10 +176,12 @@ func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error) return 0, nil } +// TearDownCluster stops the cluster and associated processes func TearDownCluster() { localCluster.Teardown() } +// TestBackup tests backup using various scenarios func TestBackup(t *testing.T) { // Run all the backup tests t.Run("TestReplicaBackup", func(t *testing.T) { @@ -228,14 +231,12 @@ func masterBackup(t *testing.T) { assert.NotNil(t, err) assert.Contains(t, output, "type MASTER cannot take backup. if you really need to do this, rerun the backup command with -allow_master") - backups := listBackups(t) - assert.Equal(t, len(backups), 0) + localCluster.VerifyBackupCount(t, shardKsName, 0) err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", "-allow_master=true", master.Alias) assert.Nil(t, err) - backups = listBackups(t) - assert.Equal(t, len(backups), 1) + backups := localCluster.VerifyBackupCount(t, shardKsName, 1) assert.Contains(t, backups[0], master.Alias) _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) @@ -383,11 +384,7 @@ func restartMasterReplica(t *testing.T) { stopAllTablets() // remove all backups - backups := listBackups(t) - for _, backup := range backups { - err := localCluster.VtctlclientProcess.ExecuteCommand("RemoveBackup", shardKsName, backup) - assert.Nil(t, err) - } + localCluster.RemoveAllBackups(t, shardKsName) // start all tablet and mysql instances var mysqlProcs []*exec.Cmd for _, tablet := range []*cluster.Vttablet{master, replica1} { @@ -489,8 +486,7 @@ func vtctlBackup(t *testing.T, tabletType string) { err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) assert.Nil(t, err) - backups := listBackups(t) - assert.Equal(t, len(backups), 1) + backups := localCluster.VerifyBackupCount(t, shardKsName, 1) _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) assert.Nil(t, err) @@ -545,12 +541,6 @@ func resetTabletDir(t *testing.T, tablet *cluster.Vttablet) { assert.Nil(t, err) } -func listBackups(t *testing.T) []string { - output, err := localCluster.ListBackups(shardKsName) - assert.Nil(t, err) - return output -} - func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []string) { // Remove the backup for _, backup := range backups { @@ -559,8 +549,7 @@ func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []st } // Now, there should not be no backup - backups = listBackups(t) - assert.Equal(t, len(backups), 0) + localCluster.VerifyBackupCount(t, shardKsName, 0) } func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string) { diff --git a/go/test/endtoend/backup/vtctlbackup/main_test.go b/go/test/endtoend/backup/vtctlbackup/main_test.go deleted file mode 100644 index aa95b77e951..00000000000 --- a/go/test/endtoend/backup/vtctlbackup/main_test.go +++ /dev/null @@ -1,158 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vtctlbackup - -import ( - "flag" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path" - "testing" - - "vitess.io/vitess/go/test/endtoend/cluster" - "vitess.io/vitess/go/test/endtoend/sharding/initialsharding" - "vitess.io/vitess/go/vt/log" -) - -var ( - master *cluster.Vttablet - replica1 *cluster.Vttablet - replica2 *cluster.Vttablet - localCluster *cluster.LocalProcessCluster - newInitDBFile string - cell = cluster.DefaultCell - hostname = "localhost" - keyspaceName = "ks" - dbPassword = "VtDbaPass" - shardKsName = fmt.Sprintf("%s/%s", keyspaceName, shardName) - dbCredentialFile string - shardName = "0" - commonTabletArg = []string{ - "-vreplication_healthcheck_topology_refresh", "1s", - "-vreplication_healthcheck_retry_delay", "1s", - "-vreplication_retry_delay", "1s", - "-degraded_threshold", "5s", - "-lock_tables_timeout", "5s", - "-watch_replication_stream", - "-enable_replication_reporter", - "-serving_state_grace_period", "1s"} -) - -func TestMain(m *testing.M) { - flag.Parse() - - exitCode, err := func() (int, error) { - localCluster = cluster.NewCluster(cell, hostname) - defer localCluster.Teardown() - - // Start topo server - err := localCluster.StartTopo() - if err != nil { - return 1, err - } - - // Start keyspace - keyspace := &cluster.Keyspace{ - Name: keyspaceName, - } - localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace) - - // update password of mysql users - dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory) - initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) - sql := string(initDb) - newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) - ioutil.WriteFile(newInitDBFile, []byte(sql), 0666) - - extraArgs := []string{"-db-credentials-file", dbCredentialFile} - commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) - - shard := cluster.Shard{ - Name: shardName, - } - - // start mysql process for all replicas and master - var mysqlProcs []*exec.Cmd - for i := 0; i < 3; i++ { - tabletType := "replica" - if i == 0 { - tabletType = "master" - } - tablet := localCluster.GetVttabletInstance(tabletType, 0, cell) - tablet.VttabletProcess = localCluster.GetVtprocessInstanceFromVttablet(tablet, shard.Name, keyspaceName) - tablet.VttabletProcess.DbPassword = dbPassword - tablet.VttabletProcess.ExtraArgs = commonTabletArg - tablet.VttabletProcess.SupportsBackup = true - tablet.VttabletProcess.EnableSemiSync = true - - tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) - tablet.MysqlctlProcess.InitDBFile = newInitDBFile - tablet.MysqlctlProcess.ExtraArgs = extraArgs - proc, err := tablet.MysqlctlProcess.StartProcess() - if err != nil { - return 1, err - } - mysqlProcs = append(mysqlProcs, proc) - - shard.Vttablets = append(shard.Vttablets, tablet) - } - for _, proc := range mysqlProcs { - if err := proc.Wait(); err != nil { - return 1, err - } - } - - // initialize tablets - master = shard.Vttablets[0] - replica1 = shard.Vttablets[1] - replica2 = shard.Vttablets[2] - - if err := localCluster.VtctlclientProcess.InitTablet(master, cell, keyspaceName, hostname, shard.Name); err != nil { - return 1, err - } - if err := localCluster.VtctlclientProcess.InitTablet(replica1, cell, keyspaceName, hostname, shard.Name); err != nil { - return 1, err - } - - // create database direct in vtTablet - for _, tablet := range []cluster.Vttablet{*master, *replica1} { - if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil { - return 1, err - } - if err := tablet.VttabletProcess.Setup(); err != nil { - return 1, err - } - } - - // initialize master and start replication - if err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shard.Name, cell, master.TabletUID); err != nil { - return 1, err - } - return m.Run(), nil - }() - - if err != nil { - log.Error(err.Error()) - os.Exit(1) - } else { - os.Exit(exitCode) - } - -} From 4acd21efa17cc55175c76e2c4d0bd8c6c160c0b6 Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Wed, 22 Jan 2020 14:19:06 +0530 Subject: [PATCH 27/50] backup-mysqlctld: mysqlctld health check changes. Signed-off-by: pradip parmar --- go/mysql/conn_params.go | 16 ++++++++++ go/test/endtoend/cluster/mysqlctld_process.go | 31 +++++++++++++------ go/test/endtoend/cluster/vttablet_process.go | 15 +-------- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/go/mysql/conn_params.go b/go/mysql/conn_params.go index d3956346612..238b978f696 100644 --- a/go/mysql/conn_params.go +++ b/go/mysql/conn_params.go @@ -59,3 +59,19 @@ func (cp *ConnParams) SslEnabled() bool { func (cp *ConnParams) EnableClientFoundRows() { cp.Flags |= CapabilityClientFoundRows } + +// NewConnParams creates ConnParams corresponds to given arguments. +func NewConnParams(port int, password, socketPath, keyspace string) ConnParams { + cp := ConnParams{ + Uname: "vt_dba", + Port: port, + UnixSocket: socketPath, + Pass: password, + } + + if keyspace != "" { + cp.DbName = "vt_" + keyspace + } + + return cp +} diff --git a/go/test/endtoend/cluster/mysqlctld_process.go b/go/test/endtoend/cluster/mysqlctld_process.go index b8bf6f5c068..fcdcc13d1df 100644 --- a/go/test/endtoend/cluster/mysqlctld_process.go +++ b/go/test/endtoend/cluster/mysqlctld_process.go @@ -36,6 +36,7 @@ type MysqlctldProcess struct { Name string Binary string LogDirectory string + Password string TabletUID int MySQLPort int InitDBFile string @@ -59,6 +60,9 @@ func (mysqlctld *MysqlctldProcess) InitDb() (err error) { // Start starts the mysqlctld and returns the error. func (mysqlctld *MysqlctldProcess) Start() error { + if mysqlctld.proc != nil { + return fmt.Errorf("process is already running") + } _ = createDirectory(mysqlctld.LogDirectory, 0700) mysqlctld.proc = exec.Command( mysqlctld.Binary, @@ -96,7 +100,7 @@ func (mysqlctld *MysqlctldProcess) Start() error { timeout := time.Now().Add(60 * time.Second) for time.Now().Before(timeout) { - if err := healthCheck(context.Background(), mysqlctld.TabletUID); err == nil { + if mysqlctld.IsHealthy() { return nil } select { @@ -126,7 +130,7 @@ func (mysqlctld *MysqlctldProcess) Stop() (err error) { mysqlctld.proc = nil return err - case <-time.After(15 * time.Second): + case <-time.After(10 * time.Second): mysqlctld.proc.Process.Kill() return <-mysqlctld.exit } @@ -185,12 +189,19 @@ func (mysqlctld *MysqlctldProcess) ExecuteCommandWithOutput(args ...string) (res return string(resultByte), err } -func healthCheck(ctx context.Context, tabletUID int) error { - params := mysql.ConnParams{ - Uname: "vt_dba", - UnixSocket: path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tabletUID), "/mysql.sock"), - } - - _, err := mysql.Connect(ctx, ¶ms) - return err +// IsHealthy gives the health status of mysql. +func (mysqlctld *MysqlctldProcess) IsHealthy() bool { + socketFile := path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", mysqlctld.TabletUID), "/mysql.sock") + params := mysql.NewConnParams(mysqlctld.MySQLPort, mysqlctld.Password, socketFile, "") + _, err := mysql.Connect(context.Background(), ¶ms) + return err == nil } + +// func fileExists(path string) bool { +// _, err := os.Stat(path) +// if os.IsNotExist(err) { +// return false +// } + +// return true +// } diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index 89cf94c07ce..4bf8c9b5eda 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -295,20 +295,7 @@ func (vttablet *VttabletProcess) CreateDB(keyspace string) error { // QueryTablet lets you execute a query in this tablet and get the result func (vttablet *VttabletProcess) QueryTablet(query string, keyspace string, useDb bool) (*sqltypes.Result, error) { - dbParams := mysql.ConnParams{ - Uname: "vt_dba", - } - if vttablet.DbPort > 0 { - dbParams.Port = vttablet.DbPort - } else { - dbParams.UnixSocket = path.Join(vttablet.Directory, "mysql.sock") - } - if useDb { - dbParams.DbName = "vt_" + keyspace - } - if vttablet.DbPassword != "" { - dbParams.Pass = vttablet.DbPassword - } + dbParams := mysql.NewConnParams(vttablet.DbPort, vttablet.DbPassword, path.Join(vttablet.Directory, "mysql.sock"), keyspace) return executeQuery(dbParams, query) } From 640bc64dbb40f25599f211789d091dab9461ddc5 Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Thu, 23 Jan 2020 12:50:53 +0530 Subject: [PATCH 28/50] backup_mysqlctld: mysqlctld teardown fixes. vttablet restart method created. Signed-off-by: pradip parmar --- go/test/endtoend/cluster/cluster_process.go | 21 +++++++ go/test/endtoend/cluster/cluster_util.go | 11 +--- go/test/endtoend/cluster/mysqlctld_process.go | 58 +++++++++---------- go/test/endtoend/cluster/vttablet_process.go | 3 + go/test/endtoend/mysqlctld/mysqlctld_test.go | 12 ++-- 5 files changed, 62 insertions(+), 43 deletions(-) diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index 40d8fc43392..64a96f091e4 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -135,6 +135,27 @@ type Vttablet struct { VttabletProcess *VttabletProcess } +// Restart restarts vttablet and mysql. +func (tablet *Vttablet) Restart() error { + if tablet.MysqlctlProcess.TabletUID|tablet.MysqlctldProcess.TabletUID == 0 { + return fmt.Errorf("no mysql process is running") + } + + if tablet.MysqlctlProcess.TabletUID > 0 { + tablet.MysqlctlProcess.Stop() + tablet.VttabletProcess.TearDown() + os.RemoveAll(tablet.VttabletProcess.Directory) + + return tablet.MysqlctlProcess.Start() + } + + tablet.MysqlctldProcess.Stop() + tablet.VttabletProcess.TearDown() + os.RemoveAll(tablet.VttabletProcess.Directory) + + return tablet.MysqlctldProcess.Start() +} + // StartTopo starts topology server func (cluster *LocalProcessCluster) StartTopo() (err error) { if cluster.Cell == "" { diff --git a/go/test/endtoend/cluster/cluster_util.go b/go/test/endtoend/cluster/cluster_util.go index 40c1f97ac42..88c051a8d7a 100644 --- a/go/test/endtoend/cluster/cluster_util.go +++ b/go/test/endtoend/cluster/cluster_util.go @@ -19,7 +19,6 @@ package cluster import ( "context" "fmt" - "os" "strings" "testing" "time" @@ -50,7 +49,7 @@ func VerifyRowsInTablet(t *testing.T, vttablet *Vttablet, ksName string, expecte timeout := time.Now().Add(10 * time.Second) for time.Now().Before(timeout) { qr, err := vttablet.VttabletProcess.QueryTablet("select * from vt_insert_test", ksName, true) - assert.Nil(t, err) + require.Nil(t, err) if len(qr.Rows) == expectedRows { return } @@ -90,12 +89,8 @@ func (cluster LocalProcessCluster) ListBackups(shardKsName string) ([]string, er } // ResetTabletDirectory transitions back to tablet state (i.e. mysql process restarts with cleaned directory and tablet is off) -func ResetTabletDirectory(tablet Vttablet) error { - tablet.MysqlctlProcess.Stop() - tablet.VttabletProcess.TearDown() - os.RemoveAll(tablet.VttabletProcess.Directory) - - return tablet.MysqlctlProcess.Start() +func ResetTabletDirectory(tablet *Vttablet) error { + return tablet.Restart() } func getTablet(tabletGrpcPort int, hostname string) *tabletpb.Tablet { diff --git a/go/test/endtoend/cluster/mysqlctld_process.go b/go/test/endtoend/cluster/mysqlctld_process.go index fcdcc13d1df..6fc75862e5c 100644 --- a/go/test/endtoend/cluster/mysqlctld_process.go +++ b/go/test/endtoend/cluster/mysqlctld_process.go @@ -23,7 +23,6 @@ import ( "os/exec" "path" "strings" - "syscall" "time" "vitess.io/vitess/go/mysql" @@ -64,39 +63,42 @@ func (mysqlctld *MysqlctldProcess) Start() error { return fmt.Errorf("process is already running") } _ = createDirectory(mysqlctld.LogDirectory, 0700) - mysqlctld.proc = exec.Command( + tempProcess := exec.Command( mysqlctld.Binary, "-log_dir", mysqlctld.LogDirectory, "-tablet_uid", fmt.Sprintf("%d", mysqlctld.TabletUID), "-mysql_port", fmt.Sprintf("%d", mysqlctld.MySQLPort), ) - mysqlctld.proc.Args = append(mysqlctld.proc.Args, mysqlctld.ExtraArgs...) + tempProcess.Args = append(tempProcess.Args, mysqlctld.ExtraArgs...) if mysqlctld.InitMysql { - mysqlctld.proc.Args = append(mysqlctld.proc.Args, + tempProcess.Args = append(tempProcess.Args, "-init_db_sql_file", mysqlctld.InitDBFile) } errFile, _ := os.Create(path.Join(mysqlctld.LogDirectory, "mysqlctld-stderr.txt")) - mysqlctld.proc.Stderr = errFile + tempProcess.Stderr = errFile - mysqlctld.proc.Env = append(mysqlctld.proc.Env, os.Environ()...) - mysqlctld.proc.Stdout = os.Stdout - mysqlctld.proc.Stderr = os.Stderr + tempProcess.Env = append(tempProcess.Env, os.Environ()...) + tempProcess.Stdout = os.Stdout + tempProcess.Stderr = os.Stderr - log.Infof("%v %v", strings.Join(mysqlctld.proc.Args, " ")) + log.Infof("%v %v", strings.Join(tempProcess.Args, " ")) - err := mysqlctld.proc.Start() + err := tempProcess.Start() if err != nil { return err } + mysqlctld.proc = tempProcess + mysqlctld.exit = make(chan error) - go func() { - mysqlctld.exit <- mysqlctld.proc.Wait() + go func(mysqlctld *MysqlctldProcess) { + err := mysqlctld.proc.Wait() mysqlctld.proc = nil - }() + mysqlctld.exit <- err + }(mysqlctld) timeout := time.Now().Add(60 * time.Second) for time.Now().Before(timeout) { @@ -116,24 +118,22 @@ func (mysqlctld *MysqlctldProcess) Start() error { } // Stop executes mysqlctld command to stop mysql instance -func (mysqlctld *MysqlctldProcess) Stop() (err error) { +func (mysqlctld *MysqlctldProcess) Stop() error { if mysqlctld.proc == nil || mysqlctld.exit == nil { return nil } - // Attempt graceful shutdown with SIGTERM first - mysqlctld.proc.Process.Signal(syscall.SIGTERM) - // mysqlctld.proc.Process.Signal(syscall.SIGTERM) - - select { - case err := <-mysqlctld.exit: - mysqlctld.proc = nil + tmpProcess := exec.Command( + "mysqlctl", + "-tablet_uid", fmt.Sprintf("%d", mysqlctld.TabletUID), + ) + tmpProcess.Args = append(tmpProcess.Args, mysqlctld.ExtraArgs...) + tmpProcess.Args = append(tmpProcess.Args, "shutdown") + err := tmpProcess.Run() + if err != nil { return err - - case <-time.After(10 * time.Second): - mysqlctld.proc.Process.Kill() - return <-mysqlctld.exit } + return <-mysqlctld.exit } // CleanupFiles clean the mysql files to make sure we can start the same process again @@ -141,9 +141,9 @@ func (mysqlctld *MysqlctldProcess) CleanupFiles(tabletUID int) { os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tabletUID))) } -// MysqlctldProcessInstance returns a Mysqlctld handle for mysqlctld process +// MysqlCtldProcessInstance returns a Mysqlctld handle for mysqlctld process // configured with the given Config. -func MysqlctldProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) *MysqlctldProcess { +func MysqlCtldProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) *MysqlctldProcess { mysqlctld := &MysqlctldProcess{ Name: "mysqlctld", Binary: "mysqlctld", @@ -158,13 +158,13 @@ func MysqlctldProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) // StartMySQLctld starts mysqlctld process func StartMySQLctld(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) error { - tablet.MysqlctldProcess = *MysqlctldProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) + tablet.MysqlctldProcess = *MysqlCtldProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) return tablet.MysqlctldProcess.Start() } // StartMySQLctldAndGetConnection create a connection to tablet mysql func StartMySQLctldAndGetConnection(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) (*mysql.Conn, error) { - tablet.MysqlctldProcess = *MysqlctldProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) + tablet.MysqlctldProcess = *MysqlCtldProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) err := tablet.MysqlctldProcess.Start() if err != nil { return nil, err diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index 676797337ca..9473bfc0431 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -295,6 +295,9 @@ func (vttablet *VttabletProcess) CreateDB(keyspace string) error { // QueryTablet lets you execute a query in this tablet and get the result func (vttablet *VttabletProcess) QueryTablet(query string, keyspace string, useDb bool) (*sqltypes.Result, error) { + if !useDb { + keyspace = "" + } dbParams := mysql.NewConnParams(vttablet.DbPort, vttablet.DbPassword, path.Join(vttablet.Directory, "mysql.sock"), keyspace) return executeQuery(dbParams, query) } diff --git a/go/test/endtoend/mysqlctld/mysqlctld_test.go b/go/test/endtoend/mysqlctld/mysqlctld_test.go index 3624aab4788..02c2889c9c4 100644 --- a/go/test/endtoend/mysqlctld/mysqlctld_test.go +++ b/go/test/endtoend/mysqlctld/mysqlctld_test.go @@ -30,8 +30,8 @@ import ( var ( clusterInstance *cluster.LocalProcessCluster - masterTablet cluster.Vttablet - replicaTablet cluster.Vttablet + masterTablet *cluster.Vttablet + replicaTablet *cluster.Vttablet hostname = "localhost" keyspaceName = "test_keyspace" shardName = "0" @@ -63,9 +63,9 @@ func TestMain(m *testing.M) { tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets for _, tablet := range tablets { if tablet.Type == "master" { - masterTablet = *tablet + masterTablet = tablet } else if tablet.Type != "rdonly" { - replicaTablet = *tablet + replicaTablet = tablet } } @@ -96,7 +96,7 @@ func initCluster(shardNames []string, totalTabletsRequired int) error { tablet.Type = "master" } // Start Mysqlctld process - tablet.MysqlctldProcess = *cluster.MysqlctldProcessInstance(tablet.TabletUID, tablet.MySQLPort, clusterInstance.TmpDirectory) + tablet.MysqlctldProcess = *cluster.MysqlCtldProcessInstance(tablet.TabletUID, tablet.MySQLPort, clusterInstance.TmpDirectory) err := tablet.MysqlctldProcess.Start() if err != nil { return err @@ -122,7 +122,7 @@ func initCluster(shardNames []string, totalTabletsRequired int) error { } for _, tablet := range shard.Vttablets { - if _, err := tablet.VttabletProcess.QueryTablet(fmt.Sprintf("create database vt_%s", keyspace.Name), keyspace.Name, false); err != nil { + if _, err := tablet.VttabletProcess.QueryTablet(fmt.Sprintf("create database vt_%s", keyspace.Name), "", false); err != nil { log.Error(err.Error()) return err } From 1a99489dc0a2b7072a7bf5ba4a75b97d4eecc88a Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Fri, 24 Jan 2020 13:46:33 +0530 Subject: [PATCH 29/50] backup-mysqlctld: mysqlctld restart fixed, backup utils refactor. Signed-off-by: pradip parmar --- .../backupmysqlctld/backup_mysqlctld_test.go | 28 ++++ .../backup/vtbackup/backup_only_test.go | 29 +--- .../backup/vtctlbackup/backup_test.go | 15 +- .../backup/vtctlbackup/backup_utils.go | 142 ++++++++++++------ .../backup/xtrabackup/xtrabackup_test.go | 11 +- .../xtrabackup_stream_test.go | 11 +- go/test/endtoend/cluster/cluster_process.go | 7 + go/test/endtoend/cluster/cluster_util.go | 5 - go/test/endtoend/cluster/mysqlctld_process.go | 43 +++--- go/test/endtoend/recovery/recovery_util.go | 5 +- 10 files changed, 163 insertions(+), 133 deletions(-) create mode 100644 go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go diff --git a/go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go b/go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go new file mode 100644 index 00000000000..f4f3edd1cd6 --- /dev/null +++ b/go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go @@ -0,0 +1,28 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctlbackup + +import ( + "testing" + + backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" +) + +// TestBackupMysqlctld - tests the backup using mysqlctld. +func TestBackupMysqlctld(t *testing.T) { + backup.TestBackup(t, backup.Mysqlctld, "", 0) +} diff --git a/go/test/endtoend/backup/vtbackup/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go index 0f987f16d21..3eec5b430a3 100644 --- a/go/test/endtoend/backup/vtbackup/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -236,7 +236,7 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS // Erase mysql/tablet dir, then start tablet with restore enabled. log.Info("restoring tablet %s", time.Now()) - resetTabletDirectory(t, *tablet, true) + tablet.ValidareTabletRestart(t) err := tablet.VttabletProcess.CreateDB(keyspaceName) assert.Nil(t, err) @@ -250,31 +250,6 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS assert.Nil(t, err) } -func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet, initMysql bool) { - - extraArgs := []string{"-db-credentials-file", dbCredentialFile} - tablet.MysqlctlProcess.ExtraArgs = extraArgs - - // Shutdown Mysql - err := tablet.MysqlctlProcess.Stop() - assert.Nil(t, err) - // Teardown Tablet - err = tablet.VttabletProcess.TearDown() - assert.Nil(t, err) - - // Empty the dir - err = os.RemoveAll(tablet.VttabletProcess.Directory) - assert.Nil(t, err) - - if initMysql { - // Init the Mysql - tablet.MysqlctlProcess.InitDBFile = newInitDBFile - err = tablet.MysqlctlProcess.Start() - assert.Nil(t, err) - } - -} - func tearDown(t *testing.T, initMysql bool) { for _, tablet := range []cluster.Vttablet{*master, *replica1, *replica2} { //Tear down Tablet @@ -283,7 +258,7 @@ func tearDown(t *testing.T, initMysql bool) { err := localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) assert.Nil(t, err) - resetTabletDirectory(t, tablet, initMysql) + tablet.ValidareTabletRestart(t) } //// reset replication diff --git a/go/test/endtoend/backup/vtctlbackup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go index f00f793b47d..6f233eafeda 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -16,18 +16,11 @@ limitations under the License. package vtctlbackup -import "testing" +import ( + "testing" +) // TestBackupMain - main tests backup using vtctl commands func TestBackupMain(t *testing.T) { - code, err := LaunchCluster(false, "", 0) - if err != nil { - t.Errorf("setup failed with status code %d", code) - } - - // Run all the backup tests - TestBackup(t) - - // Teardown the cluster - TearDownCluster() + TestBackup(t, Backup, "", 0) } diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 33c955467ab..166fd37b92f 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -34,9 +34,16 @@ import ( "vitess.io/vitess/go/vt/proto/topodata" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/endtoend/cluster" ) +const ( + ExtraBackup = iota + Backup + Mysqlctld +) + var ( master *cluster.Vttablet replica1 *cluster.Vttablet @@ -44,8 +51,6 @@ var ( localCluster *cluster.LocalProcessCluster newInitDBFile string useXtrabackup bool - xbStreamMode string - xbStripes int cell = cluster.DefaultCell hostname = "localhost" keyspaceName = "ks" @@ -72,7 +77,8 @@ var ( ) Engine=InnoDB` ) -func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error) { +// LaunchCluster : +func LaunchCluster(setupType int, streamMode string, stripes int) (int, error) { localCluster = cluster.NewCluster(cell, hostname) // Start topo server @@ -101,16 +107,14 @@ func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error) commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) // Update arguments for xtrabackup - if xtrabackup { - useXtrabackup = xtrabackup - xbStreamMode = streamMode - xbStripes = stripes + if setupType == ExtraBackup { + useXtrabackup = true xtrabackupArgs := []string{ "-backup_engine_implementation", "xtrabackup", - fmt.Sprintf("-xtrabackup_stream_mode=%s", xbStreamMode), + fmt.Sprintf("-xtrabackup_stream_mode=%s", streamMode), "-xtrabackup_user=vt_dba", - fmt.Sprintf("-xtrabackup_stripes=%d", xbStripes), + fmt.Sprintf("-xtrabackup_stripes=%d", stripes), "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), } @@ -134,14 +138,27 @@ func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error) tablet.VttabletProcess.SupportsBackup = true tablet.VttabletProcess.EnableSemiSync = true + if setupType == Mysqlctld { + tablet.MysqlctldProcess = *cluster.MysqlCtldProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) + tablet.MysqlctldProcess.InitDBFile = newInitDBFile + tablet.MysqlctldProcess.ExtraArgs = extraArgs + tablet.MysqlctldProcess.Password = tablet.VttabletProcess.DbPassword + if err := tablet.MysqlctldProcess.Start(); err != nil { + return 1, err + } + shard.Vttablets = append(shard.Vttablets, tablet) + continue + } + tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) tablet.MysqlctlProcess.InitDBFile = newInitDBFile tablet.MysqlctlProcess.ExtraArgs = extraArgs - if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { + proc, err := tablet.MysqlctlProcess.StartProcess() + if err != nil { return 1, err - } else { - mysqlProcs = append(mysqlProcs, proc) } + mysqlProcs = append(mysqlProcs, proc) + shard.Vttablets = append(shard.Vttablets, tablet) } for _, proc := range mysqlProcs { @@ -179,35 +196,60 @@ func TearDownCluster() { localCluster.Teardown() } -func TestBackup(t *testing.T) { - // Run all the backup tests - t.Run("TestReplicaBackup", func(t *testing.T) { - vtctlBackup(t, "replica") - }) - - t.Run("TestRdonlyBackup", func(t *testing.T) { - vtctlBackup(t, "rdonly") - }) - - t.Run("TestMasterBackup", func(t *testing.T) { - masterBackup(t) - }) +func TestBackup(t *testing.T, setupType int, streamMode string, stripes int) { + + testMethods := []struct { + name string + method func(t *testing.T) + }{ + { + name: "TestReplicaBackup", + method: func(t *testing.T) { + vtctlBackup(t, "replica") + }, + }, // + { + name: "TestRdonlyBackup", + method: func(t *testing.T) { + vtctlBackup(t, "rdonly") + }, + }, // + { + name: "TestMasterBackup", + method: masterBackup, + }, // + { + name: "TestMasterReplicaSameBackup", + method: masterReplicaSameBackup, + }, // + { + name: "TestRestoreOldMasterByRestart", + method: restoreOldMasterByRestart, + }, // + { + name: "TestRestoreOldMasterInPlace", + method: restoreOldMasterInPlace, + }, // + { + name: "TestTerminatedRestore", + method: terminatedRestore, + }, // + } - t.Run("TestMasterReplicaSameBackup", func(t *testing.T) { - masterReplicaSameBackup(t) - }) + // setup cluster for the testing + code, err := LaunchCluster(setupType, streamMode, stripes) + require.Nilf(t, err, "setup failed with status code %d", code) - t.Run("TestRestoreOldMasterByRestart", func(t *testing.T) { - restoreOldMasterByRestart(t) - }) + // Teardown the cluster + defer TearDownCluster() - t.Run("TestRestoreOldMasterInPlace", func(t *testing.T) { - restoreOldMasterInPlace(t) - }) + // Run all the backup tests - t.Run("TestTerminatedRestore", func(t *testing.T) { - terminatedRestore(t) - }) + for _, test := range testMethods { + fmt.Printf("test %v started \n", test.name) + pass := t.Run(test.name, test.method) + fmt.Printf("test %v completed, isPass %v\n", test.name, pass) + } } @@ -390,7 +432,12 @@ func restartMasterReplica(t *testing.T) { } // start all tablet and mysql instances var mysqlProcs []*exec.Cmd - for _, tablet := range []*cluster.Vttablet{master, replica1} { + for _, tablet := range []*cluster.Vttablet{master, replica1, replica2} { + if tablet.MysqlctldProcess.TabletUID > 0 { + err := tablet.MysqlctldProcess.Start() + require.Nilf(t, err, "error while starting mysqlctld, tabletUID %v", tablet.TabletUID) + continue + } proc, _ := tablet.MysqlctlProcess.StartProcess() mysqlProcs = append(mysqlProcs, proc) } @@ -413,6 +460,11 @@ func stopAllTablets() { var mysqlProcs []*exec.Cmd for _, tablet := range []*cluster.Vttablet{master, replica1, replica2} { tablet.VttabletProcess.TearDown() + if tablet.MysqlctldProcess.TabletUID > 0 { + tablet.MysqlctldProcess.Stop() + localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) + continue + } proc, _ := tablet.MysqlctlProcess.StopProcess() mysqlProcs = append(mysqlProcs, proc) localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) @@ -529,7 +581,7 @@ func verifyInitialReplication(t *testing.T) { // to restore a previous backup successfully regardless of this setting. func restoreWaitForBackup(t *testing.T, tabletType string) { replica2.Type = tabletType - resetTabletDir(t, replica2) + replica2.ValidareTabletRestart(t) replicaTabletArgs := commonTabletArg replicaTabletArgs = append(replicaTabletArgs, "-backup_engine_implementation", "fake_implementation") replicaTabletArgs = append(replicaTabletArgs, "-wait_for_backup_interval", "1s") @@ -540,11 +592,6 @@ func restoreWaitForBackup(t *testing.T, tabletType string) { assert.Nil(t, err) } -func resetTabletDir(t *testing.T, tablet *cluster.Vttablet) { - err := cluster.ResetTabletDirectory(*tablet) - assert.Nil(t, err) -} - func listBackups(t *testing.T) []string { output, err := localCluster.ListBackups(shardKsName) assert.Nil(t, err) @@ -564,12 +611,10 @@ func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []st } func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string) { - err := tablet.VttabletProcess.TearDown() - assert.Nil(t, err) - resetTabletDir(t, tablet) + tablet.ValidareTabletRestart(t) tablet.VttabletProcess.ServingStatus = "" - err = tablet.VttabletProcess.Setup() + err := tablet.VttabletProcess.Setup() assert.Nil(t, err) if status != "" { err = tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{status}, 25*time.Second) @@ -596,6 +641,7 @@ func terminateRestore(t *testing.T) { stopRestoreMsg := "Copying file 10" if useXtrabackup { stopRestoreMsg = "Restore: Preparing" + useXtrabackup = false } args := append([]string{"-server", localCluster.VtctlclientProcess.Server, "-alsologtostderr"}, "RestoreFromBackup", master.Alias) diff --git a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go index 4b94086bcd3..0802d116c93 100644 --- a/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go +++ b/go/test/endtoend/backup/xtrabackup/xtrabackup_test.go @@ -24,14 +24,5 @@ import ( // TestXtraBackup - tests the backup using xtrabackup func TestXtrabackup(t *testing.T) { - code, err := backup.LaunchCluster(true, "tar", 0) - if err != nil { - t.Errorf("setup failed with status code %d", code) - } - - // Run all the backup tests - backup.TestBackup(t) - - // Teardown the cluster - backup.TearDownCluster() + backup.TestBackup(t, backup.ExtraBackup, "tar", 0) } diff --git a/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go b/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go index 5e86e73719e..0f1544e8f14 100644 --- a/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go +++ b/go/test/endtoend/backup/xtrabackupstream/xtrabackup_stream_test.go @@ -24,14 +24,5 @@ import ( // TestXtrabackupStream - tests the backup using xtrabackup with xbstream mode func TestXtrabackupStream(t *testing.T) { - code, err := backup.LaunchCluster(true, "xbstream", 8) - if err != nil { - t.Errorf("setup failed with status code %d", code) - } - - // Run all the backup tests - backup.TestBackup(t) - - // Teardown the cluster - backup.TearDownCluster() + backup.TestBackup(t, backup.ExtraBackup, "xbstream", 8) } diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index 64a96f091e4..3e366ee8f1d 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -25,8 +25,10 @@ import ( "os/exec" "path" "strconv" + "testing" "time" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/vt/log" ) @@ -156,6 +158,11 @@ func (tablet *Vttablet) Restart() error { return tablet.MysqlctldProcess.Start() } +// ValidareTabletRestart restarts the tablet and validate error if there is any. +func (tablet *Vttablet) ValidareTabletRestart(t *testing.T) { + require.Nilf(t, tablet.Restart(), "tablet restart failed") +} + // StartTopo starts topology server func (cluster *LocalProcessCluster) StartTopo() (err error) { if cluster.Cell == "" { diff --git a/go/test/endtoend/cluster/cluster_util.go b/go/test/endtoend/cluster/cluster_util.go index 88c051a8d7a..7da7d2bc79c 100644 --- a/go/test/endtoend/cluster/cluster_util.go +++ b/go/test/endtoend/cluster/cluster_util.go @@ -88,11 +88,6 @@ func (cluster LocalProcessCluster) ListBackups(shardKsName string) ([]string, er return returnResult, nil } -// ResetTabletDirectory transitions back to tablet state (i.e. mysql process restarts with cleaned directory and tablet is off) -func ResetTabletDirectory(tablet *Vttablet) error { - return tablet.Restart() -} - func getTablet(tabletGrpcPort int, hostname string) *tabletpb.Tablet { portMap := make(map[string]int32) portMap["grpc"] = int32(tabletGrpcPort) diff --git a/go/test/endtoend/cluster/mysqlctld_process.go b/go/test/endtoend/cluster/mysqlctld_process.go index 6fc75862e5c..118097a3c87 100644 --- a/go/test/endtoend/cluster/mysqlctld_process.go +++ b/go/test/endtoend/cluster/mysqlctld_process.go @@ -32,17 +32,18 @@ import ( // MysqlctldProcess is a generic handle for a running mysqlctld command . // It can be spawned manually type MysqlctldProcess struct { - Name string - Binary string - LogDirectory string - Password string - TabletUID int - MySQLPort int - InitDBFile string - ExtraArgs []string - InitMysql bool - proc *exec.Cmd - exit chan error + Name string + Binary string + LogDirectory string + Password string + TabletUID int + MySQLPort int + InitDBFile string + ExtraArgs []string + process *exec.Cmd + exit chan error + InitMysql bool + exitSignalReceived bool } // InitDb executes mysqlctld command to add cell info @@ -59,7 +60,7 @@ func (mysqlctld *MysqlctldProcess) InitDb() (err error) { // Start starts the mysqlctld and returns the error. func (mysqlctld *MysqlctldProcess) Start() error { - if mysqlctld.proc != nil { + if mysqlctld.process != nil { return fmt.Errorf("process is already running") } _ = createDirectory(mysqlctld.LogDirectory, 0700) @@ -91,12 +92,16 @@ func (mysqlctld *MysqlctldProcess) Start() error { return err } - mysqlctld.proc = tempProcess + mysqlctld.process = tempProcess mysqlctld.exit = make(chan error) go func(mysqlctld *MysqlctldProcess) { - err := mysqlctld.proc.Wait() - mysqlctld.proc = nil + err := mysqlctld.process.Wait() + if !mysqlctld.exitSignalReceived { + fmt.Printf("mysqlctld stopped unexpectedly, tabletUID %v, mysql port %v, PID %v\n", mysqlctld.TabletUID, mysqlctld.MySQLPort, mysqlctld.process.Process.Pid) + } + mysqlctld.process = nil + mysqlctld.exitSignalReceived = false mysqlctld.exit <- err }(mysqlctld) @@ -119,10 +124,10 @@ func (mysqlctld *MysqlctldProcess) Start() error { // Stop executes mysqlctld command to stop mysql instance func (mysqlctld *MysqlctldProcess) Stop() error { - if mysqlctld.proc == nil || mysqlctld.exit == nil { - return nil - } - + // if mysqlctld.process == nil || mysqlctld.exit == nil { + // return nil + // } + mysqlctld.exitSignalReceived = true tmpProcess := exec.Command( "mysqlctl", "-tablet_uid", fmt.Sprintf("%d", mysqlctld.TabletUID), diff --git a/go/test/endtoend/recovery/recovery_util.go b/go/test/endtoend/recovery/recovery_util.go index c3ef62b1bc0..6b63b8d6ff9 100644 --- a/go/test/endtoend/recovery/recovery_util.go +++ b/go/test/endtoend/recovery/recovery_util.go @@ -39,11 +39,10 @@ func ExecuteQueriesUsingVtgate(t *testing.T, session *vtgateconn.VTGateSession, } func RestoreTablet(t *testing.T, localCluster *cluster.LocalProcessCluster, tablet *cluster.Vttablet, restoreKSName string, shardName string, keyspaceName string, commonTabletArg []string) { - err := cluster.ResetTabletDirectory(*tablet) - assert.Nil(t, err) + tablet.ValidareTabletRestart(t) tm := time.Now().UTC() tm.Format(time.RFC3339) - _, err = localCluster.VtctlProcess.ExecuteCommandWithOutput("CreateKeyspace", + _, err := localCluster.VtctlProcess.ExecuteCommandWithOutput("CreateKeyspace", "-keyspace_type=SNAPSHOT", "-base_keyspace="+keyspaceName, "-snapshot_time", tm.Format(time.RFC3339), restoreKSName) assert.Nil(t, err) From 5287826981fadecdd4b18a4f9e41681937c78973 Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Fri, 24 Jan 2020 17:37:44 +0530 Subject: [PATCH 30/50] backup_transform_mysqlctld: backup_transform testing using mysqlctld. Signed-off-by: pradip parmar --- .../backuptransform/backup_transform_test.go | 173 +-------- .../backup_transform_testmethods.go | 327 ++++++++++++++++++ go/test/endtoend/backuptransform/main_test.go | 135 +------- .../backup_transform_test.go | 30 ++ .../backuptransformmysqlctld/main_test.go | 27 ++ 5 files changed, 389 insertions(+), 303 deletions(-) create mode 100644 go/test/endtoend/backuptransform/backup_transform_testmethods.go create mode 100644 go/test/endtoend/backuptransformmysqlctld/backup_transform_test.go create mode 100644 go/test/endtoend/backuptransformmysqlctld/main_test.go diff --git a/go/test/endtoend/backuptransform/backup_transform_test.go b/go/test/endtoend/backuptransform/backup_transform_test.go index e8d0254ff7b..2ed4b9f8150 100644 --- a/go/test/endtoend/backuptransform/backup_transform_test.go +++ b/go/test/endtoend/backuptransform/backup_transform_test.go @@ -16,178 +16,11 @@ limitations under the License. package backuptransform -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "vitess.io/vitess/go/test/endtoend/cluster" -) - -// create query for test table creation -var vtInsertTest = `create table vt_insert_test ( - id bigint auto_increment, - msg varchar(64), - primary key (id) - ) Engine=InnoDB` +import "testing" func TestBackupTransform(t *testing.T) { - // insert data in master, validate same in slave - verifyInitialReplication(t) - - // restart the replica with transform hook parameter - replica1.VttabletProcess.TearDown() - replica1.VttabletProcess.ExtraArgs = []string{ - "-db-credentials-file", dbCredentialFile, - "-backup_storage_hook", "test_backup_transform", - "-backup_storage_compress=false", - "-restore_from_backup", - "-backup_storage_implementation", "file", - "-file_backup_storage_root", localCluster.VtctldProcess.FileBackupStorageRoot} - replica1.VttabletProcess.ServingStatus = "SERVING" - err := replica1.VttabletProcess.Setup() - assert.Nil(t, err) - - // take backup, it should not give any error - err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) - assert.Nil(t, err) - - // insert data in master - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) - assert.Nil(t, err) - - // validate backup_list, expecting 1 backup available - backups := localCluster.VerifyBackupCount(t, shardKsName, 1) - - backupLocation := localCluster.CurrentVTDATAROOT + "/backups/" + shardKsName + "/" + backups[0] - - // validate that MANIFEST has TransformHook - // every file should start with 'header' - validateManifestFile(t, backupLocation) - - // restore replica2 from backup, should not give any error - // Note: we don't need to pass in the backup_storage_transform parameter, - // as it is read from the MANIFEST. - replica2.MysqlctlProcess.ExtraArgs = []string{ - "-db-credentials-file", dbCredentialFile} - // clear replica2 - replica2.MysqlctlProcess.Stop() - os.RemoveAll(replica2.VttabletProcess.Directory) - - // start replica2 from backup - err = replica2.MysqlctlProcess.Start() - require.Nil(t, err) - err = localCluster.VtctlclientProcess.InitTablet(replica2, cell, keyspaceName, hostname, shardName) - assert.Nil(t, err) - replica2.VttabletProcess.CreateDB(keyspaceName) - replica2.VttabletProcess.ExtraArgs = []string{ - "-db-credentials-file", dbCredentialFile, - "-restore_from_backup", - "-backup_storage_implementation", "file", - "-file_backup_storage_root", localCluster.VtctldProcess.FileBackupStorageRoot} - replica2.VttabletProcess.ServingStatus = "" - err = replica2.VttabletProcess.Setup() - require.Nil(t, err) - err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) - require.Nil(t, err) - defer replica2.VttabletProcess.TearDown() - - // validate that semi-sync is enabled for replica, disable for rdOnly - if replica2.Type == "replica" { - verifyReplicationStatus(t, replica2, "ON") - } else if replica2.Type == "rdonly" { - verifyReplicationStatus(t, replica2, "OFF") - } - - // validate that new slave has all the data - cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) - - // Remove all backups - localCluster.RemoveAllBackups(t, shardKsName) - + TestBackupTransformImpl(t) } - -// TestBackupTransformError validate backup with test_backup_error -// backup_storage_hook, which should fail. func TestBackupTransformError(t *testing.T) { - // restart the replica with transform hook parameter - err := replica1.VttabletProcess.TearDown() - require.Nil(t, err) - - replica1.VttabletProcess.ExtraArgs = []string{ - "-db-credentials-file", dbCredentialFile, - "-backup_storage_hook", "test_backup_error", - "-restore_from_backup", - "-backup_storage_implementation", "file", - "-file_backup_storage_root", localCluster.VtctldProcess.FileBackupStorageRoot} - replica1.VttabletProcess.ServingStatus = "SERVING" - err = replica1.VttabletProcess.Setup() - assert.Nil(t, err) - - // create backup, it should fail - out, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("Backup", replica1.Alias) - require.NotNil(t, err) - assert.Containsf(t, out, "backup is not usable, aborting it", "unexpected error received %v", err) - - // validate there is no backup left - localCluster.VerifyBackupCount(t, shardKsName, 0) -} - -// validateManifestFile reads manifest and validates that it -// has a TransformHook, SkipCompress and FileEntries. It also -// validates that backup_files available in FileEntries have -// 'header' as their first line. -func validateManifestFile(t *testing.T, backupLocation string) { - - // reading manifest - data, err := ioutil.ReadFile(backupLocation + "/MANIFEST") - require.Nilf(t, err, "error while reading MANIFEST %v", err) - manifest := make(map[string]interface{}) - - // parsing manifest - err = json.Unmarshal(data, &manifest) - require.Nilf(t, err, "error while parsing MANIFEST %v", err) - - // validate manifest - transformHook, _ := manifest["TransformHook"] - require.Equalf(t, "test_backup_transform", transformHook, "invalid transformHook in MANIFEST") - skipCompress, _ := manifest["SkipCompress"] - assert.Equalf(t, skipCompress, true, "invalid value of skipCompress") - - // validate backup files - for i := range manifest["FileEntries"].([]interface{}) { - f, err := os.Open(fmt.Sprintf("%s/%d", backupLocation, i)) - require.Nilf(t, err, "error while opening backup_file %d: %v", i, err) - var fileHeader string - _, err = fmt.Fscanln(f, &fileHeader) - f.Close() - - require.Nilf(t, err, "error while reading backup_file %d: %v", i, err) - require.Equalf(t, "header", fileHeader, "wrong file contents for %d", i) - } - -} - -// verifyReplicationStatus validates the replication status in tablet. -func verifyReplicationStatus(t *testing.T, vttablet *cluster.Vttablet, expectedStatus string) { - status, err := vttablet.VttabletProcess.GetDBVar("rpl_semi_sync_slave_enabled", keyspaceName) - assert.Nil(t, err) - assert.Equal(t, expectedStatus, status) - status, err = vttablet.VttabletProcess.GetDBStatus("rpl_semi_sync_slave_status", keyspaceName) - assert.Nil(t, err) - assert.Equal(t, expectedStatus, status) -} - -// verifyInitialReplication creates schema in master, insert some data to master and verify the same data in replica -func verifyInitialReplication(t *testing.T) { - _, err := master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true) - assert.Nil(t, err) - _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test1')", keyspaceName, true) - assert.Nil(t, err) - cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 1) + TestBackupTransformErrorImpl(t) } diff --git a/go/test/endtoend/backuptransform/backup_transform_testmethods.go b/go/test/endtoend/backuptransform/backup_transform_testmethods.go new file mode 100644 index 00000000000..10e1dfd0cd0 --- /dev/null +++ b/go/test/endtoend/backuptransform/backup_transform_testmethods.go @@ -0,0 +1,327 @@ +package backuptransform + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/sharding/initialsharding" + "vitess.io/vitess/go/vt/log" +) + +// test main part of the testcase +var ( + master *cluster.Vttablet + replica1 *cluster.Vttablet + replica2 *cluster.Vttablet + localCluster *cluster.LocalProcessCluster + newInitDBFile string + cell = cluster.DefaultCell + hostname = "localhost" + keyspaceName = "ks" + dbPassword = "VtDbaPass" + shardKsName = fmt.Sprintf("%s/%s", keyspaceName, shardName) + dbCredentialFile string + shardName = "0" + commonTabletArg = []string{ + "-vreplication_healthcheck_topology_refresh", "1s", + "-vreplication_healthcheck_retry_delay", "1s", + "-vreplication_retry_delay", "1s", + "-degraded_threshold", "5s", + "-lock_tables_timeout", "5s", + "-watch_replication_stream", + "-enable_replication_reporter", + "-serving_state_grace_period", "1s"} +) + +func TestMainSetup(m *testing.M, useMysqlctld bool) { + flag.Parse() + + exitCode, err := func() (int, error) { + localCluster = cluster.NewCluster(cell, hostname) + defer localCluster.Teardown() + + // Start topo server + err := localCluster.StartTopo() + if err != nil { + return 1, err + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + } + localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace) + + // changing password for mysql user + dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory) + initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) + sql := string(initDb) + newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") + sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + ioutil.WriteFile(newInitDBFile, []byte(sql), 0666) + + extraArgs := []string{"-db-credentials-file", dbCredentialFile} + commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) + + shard := cluster.Shard{ + Name: shardName, + } + + // start mysql process for all replicas and master + var mysqlProcs []*exec.Cmd + for i := 0; i < 3; i++ { + tabletType := "replica" + tablet := localCluster.GetVttabletInstance(tabletType, 0, cell) + tablet.VttabletProcess = localCluster.GetVtprocessInstanceFromVttablet(tablet, shard.Name, keyspaceName) + tablet.VttabletProcess.DbPassword = dbPassword + tablet.VttabletProcess.ExtraArgs = commonTabletArg + tablet.VttabletProcess.SupportsBackup = true + tablet.VttabletProcess.EnableSemiSync = true + + if useMysqlctld { + tablet.MysqlctldProcess = *cluster.MysqlCtldProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) + tablet.MysqlctldProcess.InitDBFile = newInitDBFile + tablet.MysqlctldProcess.ExtraArgs = extraArgs + tablet.MysqlctldProcess.Password = tablet.VttabletProcess.DbPassword + err := tablet.MysqlctldProcess.Start() + if err != nil { + return 1, err + } + + shard.Vttablets = append(shard.Vttablets, tablet) + continue + } + + tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) + tablet.MysqlctlProcess.InitDBFile = newInitDBFile + tablet.MysqlctlProcess.ExtraArgs = extraArgs + proc, err := tablet.MysqlctlProcess.StartProcess() + if err != nil { + return 1, err + } + mysqlProcs = append(mysqlProcs, proc) + + shard.Vttablets = append(shard.Vttablets, tablet) + } + for _, proc := range mysqlProcs { + if err := proc.Wait(); err != nil { + return 1, err + } + } + + // initialize tablets + master = shard.Vttablets[0] + replica1 = shard.Vttablets[1] + replica2 = shard.Vttablets[2] + + for _, tablet := range []*cluster.Vttablet{master, replica1} { + if err := localCluster.VtctlclientProcess.InitTablet(tablet, cell, keyspaceName, hostname, shard.Name); err != nil { + return 1, err + } + } + + // create database for master and replica + for _, tablet := range []cluster.Vttablet{*master, *replica1} { + if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil { + return 1, err + } + if err := tablet.VttabletProcess.Setup(); err != nil { + return 1, err + } + } + + // initialize master and start replication + if err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shard.Name, cell, master.TabletUID); err != nil { + return 1, err + } + return m.Run(), nil + }() + + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } else { + os.Exit(exitCode) + } + +} + +// create query for test table creation +var vtInsertTest = `create table vt_insert_test ( + id bigint auto_increment, + msg varchar(64), + primary key (id) + ) Engine=InnoDB` + +func TestBackupTransformImpl(t *testing.T) { + // insert data in master, validate same in slave + verifyInitialReplication(t) + + // restart the replica with transform hook parameter + replica1.VttabletProcess.TearDown() + replica1.VttabletProcess.ExtraArgs = []string{ + "-db-credentials-file", dbCredentialFile, + "-backup_storage_hook", "test_backup_transform", + "-backup_storage_compress=false", + "-restore_from_backup", + "-backup_storage_implementation", "file", + "-file_backup_storage_root", localCluster.VtctldProcess.FileBackupStorageRoot} + replica1.VttabletProcess.ServingStatus = "SERVING" + err := replica1.VttabletProcess.Setup() + assert.Nil(t, err) + + // take backup, it should not give any error + err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) + assert.Nil(t, err) + + // insert data in master + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) + assert.Nil(t, err) + + // validate backup_list, expecting 1 backup available + backups := localCluster.VerifyBackupCount(t, shardKsName, 1) + + backupLocation := localCluster.CurrentVTDATAROOT + "/backups/" + shardKsName + "/" + backups[0] + + // validate that MANIFEST has TransformHook + // every file should start with 'header' + validateManifestFile(t, backupLocation) + + // restore replica2 from backup, should not give any error + // Note: we don't need to pass in the backup_storage_transform parameter, + // as it is read from the MANIFEST. + // clear replica2 + + if replica2.MysqlctlProcess.TabletUID > 0 { + replica2.MysqlctlProcess.Stop() + os.RemoveAll(replica2.VttabletProcess.Directory) + // start replica2 from backup + err = replica2.MysqlctlProcess.Start() + require.Nil(t, err) + } else { + replica2.MysqlctldProcess.Stop() + os.RemoveAll(replica2.VttabletProcess.Directory) + // start replica2 from backup + err = replica2.MysqlctldProcess.Start() + require.Nil(t, err) + } + + err = localCluster.VtctlclientProcess.InitTablet(replica2, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + replica2.VttabletProcess.CreateDB(keyspaceName) + replica2.VttabletProcess.ExtraArgs = []string{ + "-db-credentials-file", dbCredentialFile, + "-restore_from_backup", + "-backup_storage_implementation", "file", + "-file_backup_storage_root", localCluster.VtctldProcess.FileBackupStorageRoot} + replica2.VttabletProcess.ServingStatus = "" + err = replica2.VttabletProcess.Setup() + require.Nil(t, err) + err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second) + require.Nil(t, err) + defer replica2.VttabletProcess.TearDown() + + // validate that semi-sync is enabled for replica, disable for rdOnly + if replica2.Type == "replica" { + verifyReplicationStatus(t, replica2, "ON") + } else if replica2.Type == "rdonly" { + verifyReplicationStatus(t, replica2, "OFF") + } + + // validate that new slave has all the data + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + + // Remove all backups + localCluster.RemoveAllBackups(t, shardKsName) + +} + +// TestBackupTransformError validate backup with test_backup_error +// backup_storage_hook, which should fail. +func TestBackupTransformErrorImpl(t *testing.T) { + // restart the replica with transform hook parameter + err := replica1.VttabletProcess.TearDown() + require.Nil(t, err) + + replica1.VttabletProcess.ExtraArgs = []string{ + "-db-credentials-file", dbCredentialFile, + "-backup_storage_hook", "test_backup_error", + "-restore_from_backup", + "-backup_storage_implementation", "file", + "-file_backup_storage_root", localCluster.VtctldProcess.FileBackupStorageRoot} + replica1.VttabletProcess.ServingStatus = "SERVING" + err = replica1.VttabletProcess.Setup() + assert.Nil(t, err) + + // create backup, it should fail + out, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("Backup", replica1.Alias) + require.NotNil(t, err) + assert.Containsf(t, out, "backup is not usable, aborting it", "unexpected error received %v", err) + + // validate there is no backup left + localCluster.VerifyBackupCount(t, shardKsName, 0) +} + +// validateManifestFile reads manifest and validates that it +// has a TransformHook, SkipCompress and FileEntries. It also +// validates that backup_files available in FileEntries have +// 'header' as their first line. +func validateManifestFile(t *testing.T, backupLocation string) { + + // reading manifest + data, err := ioutil.ReadFile(backupLocation + "/MANIFEST") + require.Nilf(t, err, "error while reading MANIFEST %v", err) + manifest := make(map[string]interface{}) + + // parsing manifest + err = json.Unmarshal(data, &manifest) + require.Nilf(t, err, "error while parsing MANIFEST %v", err) + + // validate manifest + transformHook, _ := manifest["TransformHook"] + require.Equalf(t, "test_backup_transform", transformHook, "invalid transformHook in MANIFEST") + skipCompress, _ := manifest["SkipCompress"] + assert.Equalf(t, skipCompress, true, "invalid value of skipCompress") + + // validate backup files + for i := range manifest["FileEntries"].([]interface{}) { + f, err := os.Open(fmt.Sprintf("%s/%d", backupLocation, i)) + require.Nilf(t, err, "error while opening backup_file %d: %v", i, err) + var fileHeader string + _, err = fmt.Fscanln(f, &fileHeader) + f.Close() + + require.Nilf(t, err, "error while reading backup_file %d: %v", i, err) + require.Equalf(t, "header", fileHeader, "wrong file contents for %d", i) + } + +} + +// verifyReplicationStatus validates the replication status in tablet. +func verifyReplicationStatus(t *testing.T, vttablet *cluster.Vttablet, expectedStatus string) { + status, err := vttablet.VttabletProcess.GetDBVar("rpl_semi_sync_slave_enabled", keyspaceName) + assert.Nil(t, err) + assert.Equal(t, expectedStatus, status) + status, err = vttablet.VttabletProcess.GetDBStatus("rpl_semi_sync_slave_status", keyspaceName) + assert.Nil(t, err) + assert.Equal(t, expectedStatus, status) +} + +// verifyInitialReplication creates schema in master, insert some data to master and verify the same data in replica +func verifyInitialReplication(t *testing.T) { + _, err := master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true) + assert.Nil(t, err) + _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test1')", keyspaceName, true) + assert.Nil(t, err) + cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 1) +} diff --git a/go/test/endtoend/backuptransform/main_test.go b/go/test/endtoend/backuptransform/main_test.go index ffaeb542ace..cc07998edea 100644 --- a/go/test/endtoend/backuptransform/main_test.go +++ b/go/test/endtoend/backuptransform/main_test.go @@ -16,139 +16,8 @@ limitations under the License. package backuptransform -import ( - "flag" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path" - "testing" - - "vitess.io/vitess/go/test/endtoend/cluster" - "vitess.io/vitess/go/test/endtoend/sharding/initialsharding" - "vitess.io/vitess/go/vt/log" -) - -var ( - master *cluster.Vttablet - replica1 *cluster.Vttablet - replica2 *cluster.Vttablet - localCluster *cluster.LocalProcessCluster - newInitDBFile string - cell = cluster.DefaultCell - hostname = "localhost" - keyspaceName = "ks" - dbPassword = "VtDbaPass" - shardKsName = fmt.Sprintf("%s/%s", keyspaceName, shardName) - dbCredentialFile string - shardName = "0" - commonTabletArg = []string{ - "-vreplication_healthcheck_topology_refresh", "1s", - "-vreplication_healthcheck_retry_delay", "1s", - "-vreplication_retry_delay", "1s", - "-degraded_threshold", "5s", - "-lock_tables_timeout", "5s", - "-watch_replication_stream", - "-enable_replication_reporter", - "-serving_state_grace_period", "1s"} -) +import "testing" func TestMain(m *testing.M) { - flag.Parse() - - exitCode, err := func() (int, error) { - localCluster = cluster.NewCluster(cell, hostname) - defer localCluster.Teardown() - - // Start topo server - err := localCluster.StartTopo() - if err != nil { - return 1, err - } - - // Start keyspace - keyspace := &cluster.Keyspace{ - Name: keyspaceName, - } - localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace) - - // changing password for mysql user - dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory) - initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) - sql := string(initDb) - newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) - ioutil.WriteFile(newInitDBFile, []byte(sql), 0666) - - extraArgs := []string{"-db-credentials-file", dbCredentialFile} - commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) - - shard := cluster.Shard{ - Name: shardName, - } - - // start mysql process for all replicas and master - var mysqlProcs []*exec.Cmd - for i := 0; i < 3; i++ { - tabletType := "replica" - tablet := localCluster.GetVttabletInstance(tabletType, 0, cell) - tablet.VttabletProcess = localCluster.GetVtprocessInstanceFromVttablet(tablet, shard.Name, keyspaceName) - tablet.VttabletProcess.DbPassword = dbPassword - tablet.VttabletProcess.ExtraArgs = commonTabletArg - tablet.VttabletProcess.SupportsBackup = true - tablet.VttabletProcess.EnableSemiSync = true - - tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) - tablet.MysqlctlProcess.InitDBFile = newInitDBFile - tablet.MysqlctlProcess.ExtraArgs = extraArgs - proc, err := tablet.MysqlctlProcess.StartProcess() - if err != nil { - return 1, err - } - mysqlProcs = append(mysqlProcs, proc) - - shard.Vttablets = append(shard.Vttablets, tablet) - } - for _, proc := range mysqlProcs { - if err := proc.Wait(); err != nil { - return 1, err - } - } - - // initialize tablets - master = shard.Vttablets[0] - replica1 = shard.Vttablets[1] - replica2 = shard.Vttablets[2] - - for _, tablet := range []*cluster.Vttablet{master, replica1} { - if err := localCluster.VtctlclientProcess.InitTablet(tablet, cell, keyspaceName, hostname, shard.Name); err != nil { - return 1, err - } - } - - // create database for master and replica - for _, tablet := range []cluster.Vttablet{*master, *replica1} { - if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil { - return 1, err - } - if err := tablet.VttabletProcess.Setup(); err != nil { - return 1, err - } - } - - // initialize master and start replication - if err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shard.Name, cell, master.TabletUID); err != nil { - return 1, err - } - return m.Run(), nil - }() - - if err != nil { - log.Error(err.Error()) - os.Exit(1) - } else { - os.Exit(exitCode) - } - + TestMainSetup(m, false) } diff --git a/go/test/endtoend/backuptransformmysqlctld/backup_transform_test.go b/go/test/endtoend/backuptransformmysqlctld/backup_transform_test.go new file mode 100644 index 00000000000..a4a3c5ca842 --- /dev/null +++ b/go/test/endtoend/backuptransformmysqlctld/backup_transform_test.go @@ -0,0 +1,30 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backuptransformmysqlctld + +import ( + "testing" + + "vitess.io/vitess/go/test/endtoend/backuptransform" +) + +func TestBackupTransform(t *testing.T) { + backuptransform.TestBackupTransformImpl(t) +} +func TestBackupTransformError(t *testing.T) { + backuptransform.TestBackupTransformErrorImpl(t) +} diff --git a/go/test/endtoend/backuptransformmysqlctld/main_test.go b/go/test/endtoend/backuptransformmysqlctld/main_test.go new file mode 100644 index 00000000000..c02366c008f --- /dev/null +++ b/go/test/endtoend/backuptransformmysqlctld/main_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backuptransformmysqlctld + +import ( + "testing" + + "vitess.io/vitess/go/test/endtoend/backuptransform" +) + +func TestMain(m *testing.M) { + backuptransform.TestMainSetup(m, true) +} From 6266aa3aa6d1b02607423117838a5cce956d453e Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Wed, 29 Jan 2020 10:24:31 +0530 Subject: [PATCH 31/50] Added percona 56 new dependency Signed-off-by: Ajeet jain --- .github/workflows/cluster_endtoend.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index a4173528411..87f3b78a0ee 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -26,6 +26,7 @@ jobs: sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download + sudo apt install -y gnupg2 wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update From c57774ed30871a57bebb6b01f19de26cf9c02b8b Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Wed, 29 Jan 2020 10:29:39 +0530 Subject: [PATCH 32/50] Putting the dependency at right place Signed-off-by: Ajeet jain --- .github/workflows/cluster_endtoend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index 87f3b78a0ee..0f732b8e9be 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -26,8 +26,8 @@ jobs: sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download - sudo apt install -y gnupg2 wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update sudo apt-get install percona-xtrabackup-24 From 65ea31a69e4ac159519a2a43d8524de98ab3e8bf Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Wed, 29 Jan 2020 11:33:44 +0530 Subject: [PATCH 33/50] updated apt tp apt-get Signed-off-by: Ajeet jain --- .github/workflows/cluster_endtoend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index 0f732b8e9be..81cbfb26f81 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -27,7 +27,7 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb - sudo apt install -y gnupg2 + sudo apt-apt install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update sudo apt-get install percona-xtrabackup-24 From dee9222be5c2a5cef79c85dd2a635fc81481bbc1 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Wed, 29 Jan 2020 13:24:50 +0530 Subject: [PATCH 34/50] corrected the typo Signed-off-by: Ajeet jain --- .github/workflows/cluster_endtoend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index 81cbfb26f81..fad6e302642 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -27,7 +27,7 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb - sudo apt-apt install -y gnupg2 + sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update sudo apt-get install percona-xtrabackup-24 From af3c7f47e795c40e3e9b5fb37f54dc089f778640 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Wed, 29 Jan 2020 13:30:06 +0530 Subject: [PATCH 35/50] fixed a comma in config.json Signed-off-by: Ajeet jain --- test/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/config.json b/test/config.json index b859d8b1bb9..d00849c9733 100644 --- a/test/config.json +++ b/test/config.json @@ -198,7 +198,7 @@ "Shard": 20, "RetryMax": 0, "Tags": [] - } + }, "backup_xtrabackup_xbstream": { "File": "xtrabackup_xbstream.go", "Args": ["vitess.io/vitess/go/test/endtoend/backup/xtrabackupstream"], From 8a4b2a870a4b8b318333e5c4bd63754ca1338eba Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Thu, 30 Jan 2020 12:09:24 +0530 Subject: [PATCH 36/50] review changes. Signed-off-by: pradip parmar --- go/test/endtoend/cluster/cluster_process.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index ea00827c098..be718672ca2 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -393,8 +393,8 @@ func NewCluster(cell string, hostname string) *LocalProcessCluster { return cluster } -// RestartVtgate starts vtgate with updated configs -func (cluster *LocalProcessCluster) RestartVtgate() (err error) { +// ReStartVtgate starts vtgate with updated configs +func (cluster *LocalProcessCluster) ReStartVtgate() (err error) { err = cluster.VtgateProcess.TearDown() if err != nil { log.Error(err.Error()) From bf19133a8f6d0e339fe09d2bc29876a1c378712a Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Thu, 30 Jan 2020 13:49:14 +0530 Subject: [PATCH 37/50] test added in config. Signed-off-by: pradip parmar --- .../backupmysqlctld/backup_mysqlctld_test.go | 2 +- ....go => backup_transform_mysqlctld_test.go} | 2 +- .../backup_transform_testmethods.go | 18 +++++++++- go/test/endtoend/backuptransform/main_test.go | 2 +- test/config.json | 36 +++++++++---------- 5 files changed, 38 insertions(+), 22 deletions(-) rename go/test/endtoend/backuptransform/{backup_transform_test.go => backup_transform_mysqlctld_test.go} (95%) diff --git a/go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go b/go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go index f4f3edd1cd6..c3da4eff36f 100644 --- a/go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go +++ b/go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package vtctlbackup +package backupmysqlctld import ( "testing" diff --git a/go/test/endtoend/backuptransform/backup_transform_test.go b/go/test/endtoend/backuptransform/backup_transform_mysqlctld_test.go similarity index 95% rename from go/test/endtoend/backuptransform/backup_transform_test.go rename to go/test/endtoend/backuptransform/backup_transform_mysqlctld_test.go index 2ed4b9f8150..8a0b47a413e 100644 --- a/go/test/endtoend/backuptransform/backup_transform_test.go +++ b/go/test/endtoend/backuptransform/backup_transform_mysqlctld_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backuptransform +package backuptransformmysqlctld import "testing" diff --git a/go/test/endtoend/backuptransform/backup_transform_testmethods.go b/go/test/endtoend/backuptransform/backup_transform_testmethods.go index 10e1dfd0cd0..014defe3192 100644 --- a/go/test/endtoend/backuptransform/backup_transform_testmethods.go +++ b/go/test/endtoend/backuptransform/backup_transform_testmethods.go @@ -1,4 +1,20 @@ -package backuptransform +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backuptransformmysqlctld import ( "encoding/json" diff --git a/go/test/endtoend/backuptransform/main_test.go b/go/test/endtoend/backuptransform/main_test.go index cc07998edea..06d03d0cce2 100644 --- a/go/test/endtoend/backuptransform/main_test.go +++ b/go/test/endtoend/backuptransform/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backuptransform +package backuptransformmysqlctld import "testing" diff --git a/test/config.json b/test/config.json index d00849c9733..dc42f543469 100644 --- a/test/config.json +++ b/test/config.json @@ -1,23 +1,5 @@ { "Tests": { - "backup_mysqlctld": { - "File": "backup_mysqlctld.py", - "Args": [], - "Command": [], - "Manual": false, - "Shard": 0, - "RetryMax": 0, - "Tags": [] - }, - "backup_transform_mysqlctld": { - "File": "backup_transform_mysqlctld.py", - "Args": [], - "Command": [], - "Manual": false, - "Shard": 0, - "RetryMax": 0, - "Tags": [] - }, "check_make_parser": { "File": "", "Args": [], @@ -235,6 +217,24 @@ "RetryMax": 0, "Tags": [] }, + "backup_mysqlctld": { + "File": "backup_mysqlctld_test.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backupmysqlctld"], + "Command": [], + "Manual": false, + "Shard": 12, + "RetryMax": 0, + "Tags": [] + }, + "backup_transform_mysqlctld": { + "File": "backup_transform_mysqlctld_test.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backuptransformmysqlctld"], + "Command": [], + "Manual": false, + "Shard": 12, + "RetryMax": 0, + "Tags": [] + }, "prepare_statement": { "File": "stmt_methods_test.go", "Args": ["vitess.io/vitess/go/test/endtoend/preparestmt"], From e5f0a191d695969bbbeb6916f831dde780150d69 Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Thu, 30 Jan 2020 13:57:12 +0530 Subject: [PATCH 38/50] package name changed. Signed-off-by: pradip parmar --- .../endtoend/backuptransform/backup_transform_mysqlctld_test.go | 2 +- .../endtoend/backuptransform/backup_transform_testmethods.go | 2 +- go/test/endtoend/backuptransform/main_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/test/endtoend/backuptransform/backup_transform_mysqlctld_test.go b/go/test/endtoend/backuptransform/backup_transform_mysqlctld_test.go index 8a0b47a413e..2ed4b9f8150 100644 --- a/go/test/endtoend/backuptransform/backup_transform_mysqlctld_test.go +++ b/go/test/endtoend/backuptransform/backup_transform_mysqlctld_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backuptransformmysqlctld +package backuptransform import "testing" diff --git a/go/test/endtoend/backuptransform/backup_transform_testmethods.go b/go/test/endtoend/backuptransform/backup_transform_testmethods.go index 014defe3192..c22539c6473 100644 --- a/go/test/endtoend/backuptransform/backup_transform_testmethods.go +++ b/go/test/endtoend/backuptransform/backup_transform_testmethods.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backuptransformmysqlctld +package backuptransform import ( "encoding/json" diff --git a/go/test/endtoend/backuptransform/main_test.go b/go/test/endtoend/backuptransform/main_test.go index 06d03d0cce2..cc07998edea 100644 --- a/go/test/endtoend/backuptransform/main_test.go +++ b/go/test/endtoend/backuptransform/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backuptransformmysqlctld +package backuptransform import "testing" From ae6970bd81e1e95ef6fa0f93cbdf86a64dd338fb Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Thu, 30 Jan 2020 14:01:39 +0530 Subject: [PATCH 39/50] mysql_ctld: file name change. Signed-off-by: pradip parmar --- ...ackup_transform_mysqlctld_test.go => backup_transform_test.go} | 0 ...ackup_transform_test.go => backup_transform_mysqlctld_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename go/test/endtoend/backuptransform/{backup_transform_mysqlctld_test.go => backup_transform_test.go} (100%) rename go/test/endtoend/backuptransformmysqlctld/{backup_transform_test.go => backup_transform_mysqlctld_test.go} (100%) diff --git a/go/test/endtoend/backuptransform/backup_transform_mysqlctld_test.go b/go/test/endtoend/backuptransform/backup_transform_test.go similarity index 100% rename from go/test/endtoend/backuptransform/backup_transform_mysqlctld_test.go rename to go/test/endtoend/backuptransform/backup_transform_test.go diff --git a/go/test/endtoend/backuptransformmysqlctld/backup_transform_test.go b/go/test/endtoend/backuptransformmysqlctld/backup_transform_mysqlctld_test.go similarity index 100% rename from go/test/endtoend/backuptransformmysqlctld/backup_transform_test.go rename to go/test/endtoend/backuptransformmysqlctld/backup_transform_mysqlctld_test.go From baca2f8c626c9c9f319c57fba27d29d4d0407fc0 Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Thu, 30 Jan 2020 15:42:56 +0530 Subject: [PATCH 40/50] mysqlctld: review changes, code refactor, removed some functions. Signed-off-by: pradip parmar --- .../backup/vtbackup/backup_only_test.go | 68 +++---------------- .../backup/vtctlbackup/backup_utils.go | 6 +- go/test/endtoend/cluster/cluster_util.go | 42 ++++++++++++ go/test/endtoend/cluster/mysqlctld_process.go | 33 --------- go/test/endtoend/cluster/vttablet.go | 68 ------------------- 5 files changed, 52 insertions(+), 165 deletions(-) delete mode 100644 go/test/endtoend/cluster/vttablet.go diff --git a/go/test/endtoend/backup/vtbackup/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go index 07dec19a2f8..cf11a036b79 100644 --- a/go/test/endtoend/backup/vtbackup/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -17,17 +17,14 @@ limitations under the License. package vtbackup import ( - "bufio" "fmt" - "os" - "path" - "strings" "testing" "time" "vitess.io/vitess/go/test/endtoend/cluster" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/vt/log" ) @@ -54,9 +51,7 @@ func TestTabletInitialBackup(t *testing.T) { // - list the backups, remove them vtBackup(t, true) - backups := countBackups(t) - assert.Equal(t, 1, backups) - + localCluster.VerifyBackupCount(t, shardKsName, 1) // Initialize the tablets initTablets(t, false, false) @@ -107,10 +102,11 @@ func firstBackupTest(t *testing.T, tabletType string) { // - list the backup, remove it // Store initial backup counts - backupsCount := countBackups(t) + listbackups, err := localCluster.ListBackups(shardKsName) + require.Nil(t, err) // insert data on master, wait for slave to get it - _, err := master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true) + _, err = master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true) assert.Nil(t, err) // Add a single row with value 'test1' to the master tablet _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test1')", keyspaceName, true) @@ -125,8 +121,7 @@ func firstBackupTest(t *testing.T, tabletType string) { log.Info("done taking backup %s", time.Now()) // check that the backup shows up in the listing - backups := countBackups(t) - assert.Equal(t, backups, backupsCount+1) + localCluster.VerifyBackupCount(t, shardKsName, len(listbackups)+1) // insert more data on the master _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) @@ -154,9 +149,8 @@ func firstBackupTest(t *testing.T, tabletType string) { assert.Equal(t, "must_not", result.Rows[3][1].ToString(), "PromotionRule") } - removeBackups(t) - backups = countBackups(t) - assert.Equal(t, 0, backups) + localCluster.RemoveAllBackups(t, shardKsName) + localCluster.VerifyBackupCount(t, shardKsName, 0) } @@ -168,52 +162,6 @@ func vtBackup(t *testing.T, initialBackup bool) { assert.Nil(t, err) } -func listBackups(t *testing.T) string { - // Get a list of backup names for the current shard. - localCluster.VtctlProcess = *cluster.VtctlProcessInstance(localCluster.TopoPort, localCluster.Hostname) - backups, err := localCluster.VtctlProcess.ExecuteCommandWithOutput( - "-backup_storage_implementation", "file", - "-file_backup_storage_root", - path.Join(os.Getenv("VTDATAROOT"), "tmp", "backupstorage"), - "ListBackups", shardKsName, - ) - assert.Nil(t, err) - return backups -} - -func countBackups(t *testing.T) int { - // Count the number of backups available in current shard. - backupList := listBackups(t) - backupCount := 0 - // Counts the available backups - scanner := bufio.NewScanner(strings.NewReader(backupList)) - for scanner.Scan() { - if scanner.Text() != "" { - backupCount++ - } - } - return backupCount -} - -func removeBackups(t *testing.T) { - // Remove all the backups from the shard - backupList := listBackups(t) - - scanner := bufio.NewScanner(strings.NewReader(backupList)) - for scanner.Scan() { - if scanner.Text() != "" { - _, err := localCluster.VtctlProcess.ExecuteCommandWithOutput( - "-backup_storage_implementation", "file", - "-file_backup_storage_root", - path.Join(os.Getenv("VTDATAROOT"), "tmp", "backupstorage"), - "RemoveBackup", shardKsName, scanner.Text(), - ) - assert.Nil(t, err) - } - } - -} - func initTablets(t *testing.T, startTablet bool, initShardMaster bool) { // Initialize tablets for _, tablet := range []cluster.Vttablet{*master, *replica1} { diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 287778f02e8..0fdc62e1796 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -77,7 +77,7 @@ var ( ) Engine=InnoDB` ) -// LaunchCluster : +// LaunchCluster : starts the cluster as per given params. func LaunchCluster(setupType int, streamMode string, stripes int) (int, error) { localCluster = cluster.NewCluster(cell, hostname) @@ -246,9 +246,7 @@ func TestBackup(t *testing.T, setupType int, streamMode string, stripes int) { // Run all the backup tests for _, test := range testMethods { - fmt.Printf("test %v started \n", test.name) - pass := t.Run(test.name, test.method) - fmt.Printf("test %v completed, isPass %v\n", test.name, pass) + t.Run(test.name, test.method) } } diff --git a/go/test/endtoend/cluster/cluster_util.go b/go/test/endtoend/cluster/cluster_util.go index 93083ba810e..c140e44282e 100644 --- a/go/test/endtoend/cluster/cluster_util.go +++ b/go/test/endtoend/cluster/cluster_util.go @@ -36,6 +36,48 @@ var ( tmClient = tmc.NewClient() ) +// Vttablet stores the properties needed to start a vttablet process +type Vttablet struct { + Type string + TabletUID int + HTTPPort int + GrpcPort int + MySQLPort int + Alias string + Cell string + + // background executable processes + MysqlctlProcess MysqlctlProcess + MysqlctldProcess MysqlctldProcess + VttabletProcess *VttabletProcess +} + +// Restart restarts vttablet and mysql. +func (tablet *Vttablet) Restart() error { + if tablet.MysqlctlProcess.TabletUID|tablet.MysqlctldProcess.TabletUID == 0 { + return fmt.Errorf("no mysql process is running") + } + + if tablet.MysqlctlProcess.TabletUID > 0 { + tablet.MysqlctlProcess.Stop() + tablet.VttabletProcess.TearDown() + os.RemoveAll(tablet.VttabletProcess.Directory) + + return tablet.MysqlctlProcess.Start() + } + + tablet.MysqlctldProcess.Stop() + tablet.VttabletProcess.TearDown() + os.RemoveAll(tablet.VttabletProcess.Directory) + + return tablet.MysqlctldProcess.Start() +} + +// ValidateTabletRestart restarts the tablet and validate error if there is any. +func (tablet *Vttablet) ValidateTabletRestart(t *testing.T) { + require.Nilf(t, tablet.Restart(), "tablet restart failed") +} + // GetMasterPosition gets the master position of required vttablet func GetMasterPosition(t *testing.T, vttablet Vttablet, hostname string) (string, string) { ctx := context.Background() diff --git a/go/test/endtoend/cluster/mysqlctld_process.go b/go/test/endtoend/cluster/mysqlctld_process.go index b7658b0daab..aeb4ce6b3ba 100644 --- a/go/test/endtoend/cluster/mysqlctld_process.go +++ b/go/test/endtoend/cluster/mysqlctld_process.go @@ -161,39 +161,6 @@ func MysqlCtldProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) return mysqlctld } -// StartMySQLctld starts mysqlctld process -func StartMySQLctld(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) error { - tablet.MysqlctldProcess = *MysqlCtldProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) - return tablet.MysqlctldProcess.Start() -} - -// StartMySQLctldAndGetConnection create a connection to tablet mysql -func StartMySQLctldAndGetConnection(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) (*mysql.Conn, error) { - tablet.MysqlctldProcess = *MysqlCtldProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) - err := tablet.MysqlctldProcess.Start() - if err != nil { - return nil, err - } - - params := mysql.ConnParams{ - Uname: username, - UnixSocket: path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tablet.TabletUID), "/mysql.sock"), - } - - return mysql.Connect(ctx, ¶ms) -} - -// ExecuteCommandWithOutput executes any mysqlctld command and returns output -func (mysqlctld *MysqlctldProcess) ExecuteCommandWithOutput(args ...string) (result string, err error) { - tmpProcess := exec.Command( - mysqlctld.Binary, - args..., - ) - log.Info(fmt.Sprintf("Executing mysqlctld with arguments %v", strings.Join(tmpProcess.Args, " "))) - resultByte, err := tmpProcess.CombinedOutput() - return string(resultByte), err -} - // IsHealthy gives the health status of mysql. func (mysqlctld *MysqlctldProcess) IsHealthy() bool { socketFile := path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", mysqlctld.TabletUID), "/mysql.sock") diff --git a/go/test/endtoend/cluster/vttablet.go b/go/test/endtoend/cluster/vttablet.go deleted file mode 100644 index 60ae579825a..00000000000 --- a/go/test/endtoend/cluster/vttablet.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -*/ - -package cluster - -import ( - "fmt" - "os" - "testing" - - "github.com/stretchr/testify/require" -) - -// Vttablet stores the properties needed to start a vttablet process -type Vttablet struct { - Type string - TabletUID int - HTTPPort int - GrpcPort int - MySQLPort int - Alias string - Cell string - - // background executable processes - MysqlctlProcess MysqlctlProcess - MysqlctldProcess MysqlctldProcess - VttabletProcess *VttabletProcess -} - -// Restart restarts vttablet and mysql. -func (tablet *Vttablet) Restart() error { - if tablet.MysqlctlProcess.TabletUID|tablet.MysqlctldProcess.TabletUID == 0 { - return fmt.Errorf("no mysql process is running") - } - - if tablet.MysqlctlProcess.TabletUID > 0 { - tablet.MysqlctlProcess.Stop() - tablet.VttabletProcess.TearDown() - os.RemoveAll(tablet.VttabletProcess.Directory) - - return tablet.MysqlctlProcess.Start() - } - - tablet.MysqlctldProcess.Stop() - tablet.VttabletProcess.TearDown() - os.RemoveAll(tablet.VttabletProcess.Directory) - - return tablet.MysqlctldProcess.Start() -} - -// ValidateTabletRestart restarts the tablet and validate error if there is any. -func (tablet *Vttablet) ValidateTabletRestart(t *testing.T) { - require.Nilf(t, tablet.Restart(), "tablet restart failed") -} From dbed9290b8f0d08724cadfeb772d9cf02be5e13a Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Fri, 31 Jan 2020 10:23:15 +0530 Subject: [PATCH 41/50] backup_transform: refator. Signed-off-by: pradip parmar --- .github/workflows/cluster_endtoend.yml | 2 +- .../backup_mysqlctld_test.go | 2 +- .../transform}/backup_transform_test.go | 2 +- .../backup_transform_testmethods.go | 2 +- .../transform}/main_test.go | 2 +- .../backup_transform_mysqlctld_test.go | 8 +-- .../transformmysqlctld}/main_test.go | 6 +- .../backup/vtbackup/backup_only_test.go | 30 ++++++++- test/config.json | 65 +++++++++++-------- 9 files changed, 78 insertions(+), 41 deletions(-) rename go/test/endtoend/backup/{backupmysqlctld => mysqlctld}/backup_mysqlctld_test.go (97%) rename go/test/endtoend/{backuptransform => backup/transform}/backup_transform_test.go (96%) rename go/test/endtoend/{backuptransform => backup/transform}/backup_transform_testmethods.go (99%) rename go/test/endtoend/{backuptransform => backup/transform}/main_test.go (96%) rename go/test/endtoend/{backuptransformmysqlctld => backup/transformmysqlctld}/backup_transform_mysqlctld_test.go (79%) rename go/test/endtoend/{backuptransformmysqlctld => backup/transformmysqlctld}/main_test.go (83%) diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index fad6e302642..c6013c610a6 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20] + name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21] steps: - name: Set up Go diff --git a/go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go b/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go similarity index 97% rename from go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go rename to go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go index c3da4eff36f..2ee11103c72 100644 --- a/go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go +++ b/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backupmysqlctld +package mysqlctld import ( "testing" diff --git a/go/test/endtoend/backuptransform/backup_transform_test.go b/go/test/endtoend/backup/transform/backup_transform_test.go similarity index 96% rename from go/test/endtoend/backuptransform/backup_transform_test.go rename to go/test/endtoend/backup/transform/backup_transform_test.go index 2ed4b9f8150..3b079127090 100644 --- a/go/test/endtoend/backuptransform/backup_transform_test.go +++ b/go/test/endtoend/backup/transform/backup_transform_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backuptransform +package transform import "testing" diff --git a/go/test/endtoend/backuptransform/backup_transform_testmethods.go b/go/test/endtoend/backup/transform/backup_transform_testmethods.go similarity index 99% rename from go/test/endtoend/backuptransform/backup_transform_testmethods.go rename to go/test/endtoend/backup/transform/backup_transform_testmethods.go index c22539c6473..227cd35e023 100644 --- a/go/test/endtoend/backuptransform/backup_transform_testmethods.go +++ b/go/test/endtoend/backup/transform/backup_transform_testmethods.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backuptransform +package transform import ( "encoding/json" diff --git a/go/test/endtoend/backuptransform/main_test.go b/go/test/endtoend/backup/transform/main_test.go similarity index 96% rename from go/test/endtoend/backuptransform/main_test.go rename to go/test/endtoend/backup/transform/main_test.go index cc07998edea..7d043af30d1 100644 --- a/go/test/endtoend/backuptransform/main_test.go +++ b/go/test/endtoend/backup/transform/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backuptransform +package transform import "testing" diff --git a/go/test/endtoend/backuptransformmysqlctld/backup_transform_mysqlctld_test.go b/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go similarity index 79% rename from go/test/endtoend/backuptransformmysqlctld/backup_transform_mysqlctld_test.go rename to go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go index a4a3c5ca842..a3746882b53 100644 --- a/go/test/endtoend/backuptransformmysqlctld/backup_transform_mysqlctld_test.go +++ b/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go @@ -14,17 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backuptransformmysqlctld +package transformmysqlctld import ( "testing" - "vitess.io/vitess/go/test/endtoend/backuptransform" + "vitess.io/vitess/go/test/endtoend/backup/transform" ) func TestBackupTransform(t *testing.T) { - backuptransform.TestBackupTransformImpl(t) + transform.TestBackupTransformImpl(t) } func TestBackupTransformError(t *testing.T) { - backuptransform.TestBackupTransformErrorImpl(t) + transform.TestBackupTransformErrorImpl(t) } diff --git a/go/test/endtoend/backuptransformmysqlctld/main_test.go b/go/test/endtoend/backup/transformmysqlctld/main_test.go similarity index 83% rename from go/test/endtoend/backuptransformmysqlctld/main_test.go rename to go/test/endtoend/backup/transformmysqlctld/main_test.go index c02366c008f..d9abfc47897 100644 --- a/go/test/endtoend/backuptransformmysqlctld/main_test.go +++ b/go/test/endtoend/backup/transformmysqlctld/main_test.go @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backuptransformmysqlctld +package transformmysqlctld import ( "testing" - "vitess.io/vitess/go/test/endtoend/backuptransform" + "vitess.io/vitess/go/test/endtoend/backup/transform" ) func TestMain(m *testing.M) { - backuptransform.TestMainSetup(m, true) + transform.TestMainSetup(m, true) } diff --git a/go/test/endtoend/backup/vtbackup/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go index cf11a036b79..ac1083bc11b 100644 --- a/go/test/endtoend/backup/vtbackup/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -18,6 +18,7 @@ package vtbackup import ( "fmt" + "os" "testing" "time" @@ -185,7 +186,7 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS // Erase mysql/tablet dir, then start tablet with restore enabled. log.Info("restoring tablet %s", time.Now()) - tablet.ValidateTabletRestart(t) + resetTabletDirectory(t, *tablet, true) err := tablet.VttabletProcess.CreateDB(keyspaceName) assert.Nil(t, err) @@ -199,6 +200,31 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS assert.Nil(t, err) } +func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet, initMysql bool) { + + extraArgs := []string{"-db-credentials-file", dbCredentialFile} + tablet.MysqlctlProcess.ExtraArgs = extraArgs + + // Shutdown Mysql + err := tablet.MysqlctlProcess.Stop() + assert.Nil(t, err) + // Teardown Tablet + err = tablet.VttabletProcess.TearDown() + assert.Nil(t, err) + + // Empty the dir + err = os.RemoveAll(tablet.VttabletProcess.Directory) + assert.Nil(t, err) + + if initMysql { + // Init the Mysql + tablet.MysqlctlProcess.InitDBFile = newInitDBFile + err = tablet.MysqlctlProcess.Start() + assert.Nil(t, err) + } + +} + func tearDown(t *testing.T, initMysql bool) { // reset replication promoteSlaveCommands := "STOP SLAVE; RESET SLAVE ALL; RESET MASTER;" @@ -223,6 +249,6 @@ func tearDown(t *testing.T, initMysql bool) { err := localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias) assert.Nil(t, err) - tablet.ValidateTabletRestart(t) + resetTabletDirectory(t, tablet, initMysql) } } diff --git a/test/config.json b/test/config.json index dc42f543469..7dd5e856855 100644 --- a/test/config.json +++ b/test/config.json @@ -163,6 +163,15 @@ "RetryMax": 0, "Tags": [] }, + "backup_mysqlctld": { + "File": "backup_mysqlctld_test.go", + "Args": ["vitess.io/vitess/go/test/endtoend/mysqlctld"], + "Command": [], + "Manual": false, + "Shard": 21, + "RetryMax": 0, + "Tags": [] + }, "backup_only": { "File": "backup_only.go", "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtbackup"], @@ -172,6 +181,24 @@ "RetryMax": 0, "Tags": [] }, + "backup_transform": { + "File": "backup_transform_test.go", + "Args": ["vitess.io/vitess/go/test/endtoend/transform"], + "Command": [], + "Manual": false, + "Shard": 21, + "RetryMax": 0, + "Tags": [] + }, + "backup_transform_mysqlctld": { + "File": "backup_transform_mysqlctld_test.go", + "Args": ["vitess.io/vitess/go/test/endtoend/transformmysqlctld"], + "Command": [], + "Manual": false, + "Shard": 21, + "RetryMax": 0, + "Tags": [] + }, "backup_xtrabackup": { "File": "xtrabackup.go", "Args": ["vitess.io/vitess/go/test/endtoend/backup/xtrabackup"], @@ -208,33 +235,6 @@ "RetryMax": 0, "Tags": [] }, - "backup_transform": { - "File": "backup_transform_test.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backuptransform"], - "Command": [], - "Manual": false, - "Shard": 12, - "RetryMax": 0, - "Tags": [] - }, - "backup_mysqlctld": { - "File": "backup_mysqlctld_test.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backupmysqlctld"], - "Command": [], - "Manual": false, - "Shard": 12, - "RetryMax": 0, - "Tags": [] - }, - "backup_transform_mysqlctld": { - "File": "backup_transform_mysqlctld_test.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backuptransformmysqlctld"], - "Command": [], - "Manual": false, - "Shard": 12, - "RetryMax": 0, - "Tags": [] - }, "prepare_statement": { "File": "stmt_methods_test.go", "Args": ["vitess.io/vitess/go/test/endtoend/preparestmt"], @@ -329,6 +329,17 @@ "site_test" ] }, + "mysqlctld": { + "File": "mysqlctld_test.go", + "Args": ["vitess.io/vitess/go/test/endtoend/mysqlctld"], + "Command": [], + "Manual": false, + "Shard": 12, + "RetryMax": 0, + "Tags": [ + "site_test" + ] + }, "recovery": { "File": "recovery.go", "Args": ["vitess.io/vitess/go/test/endtoend/recovery/unshardedrecovery"], From ac01391bb593bea7c8f8456d62aa65605a2024cd Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Fri, 31 Jan 2020 12:01:46 +0530 Subject: [PATCH 42/50] backup_mysqlctld: config changes. Signed-off-by: pradip parmar --- test/config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/config.json b/test/config.json index 7dd5e856855..498f96a35d5 100644 --- a/test/config.json +++ b/test/config.json @@ -165,7 +165,7 @@ }, "backup_mysqlctld": { "File": "backup_mysqlctld_test.go", - "Args": ["vitess.io/vitess/go/test/endtoend/mysqlctld"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/mysqlctld"], "Command": [], "Manual": false, "Shard": 21, @@ -183,7 +183,7 @@ }, "backup_transform": { "File": "backup_transform_test.go", - "Args": ["vitess.io/vitess/go/test/endtoend/transform"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/transform"], "Command": [], "Manual": false, "Shard": 21, @@ -192,7 +192,7 @@ }, "backup_transform_mysqlctld": { "File": "backup_transform_mysqlctld_test.go", - "Args": ["vitess.io/vitess/go/test/endtoend/transformmysqlctld"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/transformmysqlctld"], "Command": [], "Manual": false, "Shard": 21, From 49189bec520c8f688311a6c03d6d3da2309dc900 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Fri, 31 Jan 2020 12:38:10 +0530 Subject: [PATCH 43/50] restore vtBackup test Signed-off-by: Ajeet jain --- .../backup/vtbackup/backup_only_test.go | 58 +++++++++++++++++-- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/go/test/endtoend/backup/vtbackup/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go index ac1083bc11b..5a6de0d8a56 100644 --- a/go/test/endtoend/backup/vtbackup/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -19,13 +19,16 @@ package vtbackup import ( "fmt" "os" + "path" + "strings" "testing" "time" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/test/endtoend/cluster" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/vt/log" ) @@ -52,7 +55,8 @@ func TestTabletInitialBackup(t *testing.T) { // - list the backups, remove them vtBackup(t, true) - localCluster.VerifyBackupCount(t, shardKsName, 1) + verifyBackupCount(t, shardKsName, 1) + // Initialize the tablets initTablets(t, false, false) @@ -103,7 +107,7 @@ func firstBackupTest(t *testing.T, tabletType string) { // - list the backup, remove it // Store initial backup counts - listbackups, err := localCluster.ListBackups(shardKsName) + backups, err := listBackups(shardKsName) require.Nil(t, err) // insert data on master, wait for slave to get it @@ -122,7 +126,7 @@ func firstBackupTest(t *testing.T, tabletType string) { log.Info("done taking backup %s", time.Now()) // check that the backup shows up in the listing - localCluster.VerifyBackupCount(t, shardKsName, len(listbackups)+1) + verifyBackupCount(t, shardKsName, len(backups)+1) // insert more data on the master _, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) @@ -150,8 +154,8 @@ func firstBackupTest(t *testing.T, tabletType string) { assert.Equal(t, "must_not", result.Rows[3][1].ToString(), "PromotionRule") } - localCluster.RemoveAllBackups(t, shardKsName) - localCluster.VerifyBackupCount(t, shardKsName, 0) + removeBackups(t) + verifyBackupCount(t, shardKsName, 0) } @@ -163,6 +167,48 @@ func vtBackup(t *testing.T, initialBackup bool) { assert.Nil(t, err) } +func verifyBackupCount(t *testing.T, shardKsName string, expected int) []string { + backups, err := listBackups(shardKsName) + assert.Nil(t, err) + assert.Equalf(t, expected, len(backups), "invalid number of backups") + return backups +} + +func listBackups(shardKsName string) ([]string, error) { + backups, err := localCluster.VtctlProcess.ExecuteCommandWithOutput( + "-backup_storage_implementation", "file", + "-file_backup_storage_root", + path.Join(os.Getenv("VTDATAROOT"), "tmp", "backupstorage"), + "ListBackups", shardKsName, + ) + if err != nil { + return nil, err + } + result := strings.Split(backups, "\n") + var returnResult []string + for _, str := range result { + if str != "" { + returnResult = append(returnResult, str) + } + } + return returnResult, nil +} + +func removeBackups(t *testing.T) { + // Remove all the backups from the shard + backups, err := listBackups(shardKsName) + assert.Nil(t, err) + for _, backup := range backups { + _, err := localCluster.VtctlProcess.ExecuteCommandWithOutput( + "-backup_storage_implementation", "file", + "-file_backup_storage_root", + path.Join(os.Getenv("VTDATAROOT"), "tmp", "backupstorage"), + "RemoveBackup", shardKsName, backup, + ) + assert.Nil(t, err) + } +} + func initTablets(t *testing.T, startTablet bool, initShardMaster bool) { // Initialize tablets for _, tablet := range []cluster.Vttablet{*master, *replica1} { From 05ea4d449da20bf7453616238ef2b99af15665d8 Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Fri, 31 Jan 2020 13:19:51 +0530 Subject: [PATCH 44/50] redistribute travis tests Signed-off-by: Ajeet jain --- test/config.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/config.json b/test/config.json index 498f96a35d5..60376af8ea4 100644 --- a/test/config.json +++ b/test/config.json @@ -19,7 +19,7 @@ "java_test" ], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [] }, @@ -28,7 +28,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 1, "RetryMax": 0, "Tags": [ "worker_test" @@ -52,7 +52,7 @@ "test/client_test.sh" ], "Manual": false, - "Shard": 3, + "Shard": 2, "RetryMax": 0, "Tags": [] }, @@ -101,7 +101,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 2, "RetryMax": 0, "Tags": [] }, @@ -110,7 +110,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 2, "RetryMax": 0, "Tags": [] }, @@ -119,7 +119,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 1, "RetryMax": 0, "Tags": [ "webdriver" From 4ef2a1fafb9b728340d4fdd0be4f6cd7418d339b Mon Sep 17 00:00:00 2001 From: Ajeet jain Date: Fri, 31 Jan 2020 13:37:13 +0530 Subject: [PATCH 45/50] updated shard matrix for transform test Signed-off-by: Ajeet jain --- test/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/config.json b/test/config.json index 60376af8ea4..7298e383e08 100644 --- a/test/config.json +++ b/test/config.json @@ -186,7 +186,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/backup/transform"], "Command": [], "Manual": false, - "Shard": 21, + "Shard": 19, "RetryMax": 0, "Tags": [] }, From f8c103d9206c7710edf477282f1b3271ab535c63 Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Fri, 31 Jan 2020 14:52:17 +0530 Subject: [PATCH 46/50] mysqlctld: mysqlctld teardown issue resolved. Signed-off-by: pradip parmar --- .../backup/transform/backup_transform_test.go | 4 +++ ...stmethods.go => backup_transform_utils.go} | 11 ++++---- .../endtoend/backup/transform/main_test.go | 23 ---------------- .../backup_transform_mysqlctld_test.go | 4 +++ .../backup/transformmysqlctld/main_test.go | 27 ------------------- 5 files changed, 14 insertions(+), 55 deletions(-) rename go/test/endtoend/backup/transform/{backup_transform_testmethods.go => backup_transform_utils.go} (99%) delete mode 100644 go/test/endtoend/backup/transform/main_test.go delete mode 100644 go/test/endtoend/backup/transformmysqlctld/main_test.go diff --git a/go/test/endtoend/backup/transform/backup_transform_test.go b/go/test/endtoend/backup/transform/backup_transform_test.go index 3b079127090..071f7d536e3 100644 --- a/go/test/endtoend/backup/transform/backup_transform_test.go +++ b/go/test/endtoend/backup/transform/backup_transform_test.go @@ -18,6 +18,10 @@ package transform import "testing" +func TestMain(m *testing.M) { + TestMainSetup(m, false) +} + func TestBackupTransform(t *testing.T) { TestBackupTransformImpl(t) } diff --git a/go/test/endtoend/backup/transform/backup_transform_testmethods.go b/go/test/endtoend/backup/transform/backup_transform_utils.go similarity index 99% rename from go/test/endtoend/backup/transform/backup_transform_testmethods.go rename to go/test/endtoend/backup/transform/backup_transform_utils.go index 227cd35e023..8b1c43482f1 100644 --- a/go/test/endtoend/backup/transform/backup_transform_testmethods.go +++ b/go/test/endtoend/backup/transform/backup_transform_utils.go @@ -75,9 +75,14 @@ func TestMainSetup(m *testing.M, useMysqlctld bool) { // Start keyspace keyspace := &cluster.Keyspace{ Name: keyspaceName, + Shards: []cluster.Shard{ + { + Name: shardName, + }, + }, } localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace) - + shard := &keyspace.Shards[0] // changing password for mysql user dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory) initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) @@ -89,10 +94,6 @@ func TestMainSetup(m *testing.M, useMysqlctld bool) { extraArgs := []string{"-db-credentials-file", dbCredentialFile} commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) - shard := cluster.Shard{ - Name: shardName, - } - // start mysql process for all replicas and master var mysqlProcs []*exec.Cmd for i := 0; i < 3; i++ { diff --git a/go/test/endtoend/backup/transform/main_test.go b/go/test/endtoend/backup/transform/main_test.go deleted file mode 100644 index 7d043af30d1..00000000000 --- a/go/test/endtoend/backup/transform/main_test.go +++ /dev/null @@ -1,23 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package transform - -import "testing" - -func TestMain(m *testing.M) { - TestMainSetup(m, false) -} diff --git a/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go b/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go index a3746882b53..fa8501fd66d 100644 --- a/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go +++ b/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go @@ -22,6 +22,10 @@ import ( "vitess.io/vitess/go/test/endtoend/backup/transform" ) +func TestMain(m *testing.M) { + transform.TestMainSetup(m, true) +} + func TestBackupTransform(t *testing.T) { transform.TestBackupTransformImpl(t) } diff --git a/go/test/endtoend/backup/transformmysqlctld/main_test.go b/go/test/endtoend/backup/transformmysqlctld/main_test.go deleted file mode 100644 index d9abfc47897..00000000000 --- a/go/test/endtoend/backup/transformmysqlctld/main_test.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package transformmysqlctld - -import ( - "testing" - - "vitess.io/vitess/go/test/endtoend/backup/transform" -) - -func TestMain(m *testing.M) { - transform.TestMainSetup(m, true) -} From 4fa21f5bcbe41272d499fb60a18d1ace74695f0c Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Fri, 31 Jan 2020 14:52:17 +0530 Subject: [PATCH 47/50] mysqlctld: mysqlctld teardown issue resolved. Signed-off-by: pradip parmar --- .../backup/transform/backup_transform_test.go | 4 +++ ...stmethods.go => backup_transform_utils.go} | 11 ++++---- .../endtoend/backup/transform/main_test.go | 23 ---------------- .../backup_transform_mysqlctld_test.go | 4 +++ .../backup/transformmysqlctld/main_test.go | 27 ------------------- 5 files changed, 14 insertions(+), 55 deletions(-) rename go/test/endtoend/backup/transform/{backup_transform_testmethods.go => backup_transform_utils.go} (99%) delete mode 100644 go/test/endtoend/backup/transform/main_test.go delete mode 100644 go/test/endtoend/backup/transformmysqlctld/main_test.go diff --git a/go/test/endtoend/backup/transform/backup_transform_test.go b/go/test/endtoend/backup/transform/backup_transform_test.go index 3b079127090..071f7d536e3 100644 --- a/go/test/endtoend/backup/transform/backup_transform_test.go +++ b/go/test/endtoend/backup/transform/backup_transform_test.go @@ -18,6 +18,10 @@ package transform import "testing" +func TestMain(m *testing.M) { + TestMainSetup(m, false) +} + func TestBackupTransform(t *testing.T) { TestBackupTransformImpl(t) } diff --git a/go/test/endtoend/backup/transform/backup_transform_testmethods.go b/go/test/endtoend/backup/transform/backup_transform_utils.go similarity index 99% rename from go/test/endtoend/backup/transform/backup_transform_testmethods.go rename to go/test/endtoend/backup/transform/backup_transform_utils.go index 227cd35e023..8b1c43482f1 100644 --- a/go/test/endtoend/backup/transform/backup_transform_testmethods.go +++ b/go/test/endtoend/backup/transform/backup_transform_utils.go @@ -75,9 +75,14 @@ func TestMainSetup(m *testing.M, useMysqlctld bool) { // Start keyspace keyspace := &cluster.Keyspace{ Name: keyspaceName, + Shards: []cluster.Shard{ + { + Name: shardName, + }, + }, } localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace) - + shard := &keyspace.Shards[0] // changing password for mysql user dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory) initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) @@ -89,10 +94,6 @@ func TestMainSetup(m *testing.M, useMysqlctld bool) { extraArgs := []string{"-db-credentials-file", dbCredentialFile} commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile) - shard := cluster.Shard{ - Name: shardName, - } - // start mysql process for all replicas and master var mysqlProcs []*exec.Cmd for i := 0; i < 3; i++ { diff --git a/go/test/endtoend/backup/transform/main_test.go b/go/test/endtoend/backup/transform/main_test.go deleted file mode 100644 index 7d043af30d1..00000000000 --- a/go/test/endtoend/backup/transform/main_test.go +++ /dev/null @@ -1,23 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package transform - -import "testing" - -func TestMain(m *testing.M) { - TestMainSetup(m, false) -} diff --git a/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go b/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go index a3746882b53..fa8501fd66d 100644 --- a/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go +++ b/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go @@ -22,6 +22,10 @@ import ( "vitess.io/vitess/go/test/endtoend/backup/transform" ) +func TestMain(m *testing.M) { + transform.TestMainSetup(m, true) +} + func TestBackupTransform(t *testing.T) { transform.TestBackupTransformImpl(t) } diff --git a/go/test/endtoend/backup/transformmysqlctld/main_test.go b/go/test/endtoend/backup/transformmysqlctld/main_test.go deleted file mode 100644 index d9abfc47897..00000000000 --- a/go/test/endtoend/backup/transformmysqlctld/main_test.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package transformmysqlctld - -import ( - "testing" - - "vitess.io/vitess/go/test/endtoend/backup/transform" -) - -func TestMain(m *testing.M) { - transform.TestMainSetup(m, true) -} From 3b1e455c424405c60753961755b1b71be4d9f075 Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Thu, 6 Feb 2020 10:59:57 +0530 Subject: [PATCH 48/50] mysql-ctld: process teardown changes. Signed-off-by: pradip parmar --- go/test/endtoend/cluster/vtctld_process.go | 2 +- go/test/endtoend/cluster/vtgate_process.go | 2 +- go/test/endtoend/cluster/vttablet_process.go | 2 +- go/test/endtoend/cluster/vtworker_process.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/test/endtoend/cluster/vtctld_process.go b/go/test/endtoend/cluster/vtctld_process.go index 4bb4ef1bd99..89a3ef799d6 100644 --- a/go/test/endtoend/cluster/vtctld_process.go +++ b/go/test/endtoend/cluster/vtctld_process.go @@ -87,7 +87,6 @@ func (vtctld *VtctldProcess) Setup(cell string, extraArgs ...string) (err error) vtctld.exit = make(chan error) go func() { vtctld.exit <- vtctld.proc.Wait() - vtctld.proc = nil }() timeout := time.Now().Add(60 * time.Second) @@ -141,6 +140,7 @@ func (vtctld *VtctldProcess) TearDown() error { case <-time.After(10 * time.Second): vtctld.proc.Process.Kill() + vtctld.proc = nil return <-vtctld.exit } } diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index f9c6857b1d9..12dad980510 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -100,7 +100,6 @@ func (vtgate *VtgateProcess) Setup() (err error) { go func() { if vtgate.proc != nil { vtgate.exit <- vtgate.proc.Wait() - vtgate.proc = nil } }() @@ -195,6 +194,7 @@ func (vtgate *VtgateProcess) TearDown() error { case <-time.After(10 * time.Second): vtgate.proc.Process.Kill() + vtgate.proc = nil return <-vtgate.exit } } diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index d69384ca238..c919f97555a 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -125,7 +125,6 @@ func (vttablet *VttabletProcess) Setup() (err error) { go func() { if vttablet.proc != nil { vttablet.exit <- vttablet.proc.Wait() - vttablet.proc = nil } }() @@ -282,6 +281,7 @@ func (vttablet *VttabletProcess) TearDown() error { case <-time.After(10 * time.Second): vttablet.proc.Process.Kill() + vttablet.proc = nil return <-vttablet.exit } } diff --git a/go/test/endtoend/cluster/vtworker_process.go b/go/test/endtoend/cluster/vtworker_process.go index 3fb600655ed..46ab0fb715b 100644 --- a/go/test/endtoend/cluster/vtworker_process.go +++ b/go/test/endtoend/cluster/vtworker_process.go @@ -85,7 +85,6 @@ func (vtworker *VtworkerProcess) Setup(cell string) (err error) { vtworker.exit = make(chan error) go func() { vtworker.exit <- vtworker.proc.Wait() - vtworker.proc = nil }() timeout := time.Now().Add(60 * time.Second) @@ -132,6 +131,7 @@ func (vtworker *VtworkerProcess) TearDown() error { case <-time.After(10 * time.Second): vtworker.proc.Process.Kill() + vtworker.proc = nil return <-vtworker.exit } } From 6700a4f91712b245fcf3382cd5a040a11f3b9154 Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Thu, 6 Feb 2020 13:34:30 +0530 Subject: [PATCH 49/50] review changes and newConnection method modified. Signed-off-by: pradip parmar --- .../backup_transform_mysqlctld_test.go | 2 +- go/test/endtoend/cluster/cluster_process.go | 16 ++++++++++++++++ go/test/endtoend/cluster/cluster_util.go | 19 +++---------------- go/test/endtoend/mysqlctl/mysqlctl_test.go | 2 +- go/test/endtoend/mysqlctld/mysqlctld_test.go | 2 +- test/config.json | 2 +- 6 files changed, 23 insertions(+), 20 deletions(-) rename go/test/endtoend/backup/{transformmysqlctld => transform/mysqlctld}/backup_transform_mysqlctld_test.go (97%) diff --git a/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go b/go/test/endtoend/backup/transform/mysqlctld/backup_transform_mysqlctld_test.go similarity index 97% rename from go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go rename to go/test/endtoend/backup/transform/mysqlctld/backup_transform_mysqlctld_test.go index fa8501fd66d..0a3b11227da 100644 --- a/go/test/endtoend/backup/transformmysqlctld/backup_transform_mysqlctld_test.go +++ b/go/test/endtoend/backup/transform/mysqlctld/backup_transform_mysqlctld_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package transformmysqlctld +package mysqlctld import ( "testing" diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index be718672ca2..d05e3fb5d75 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -80,6 +80,22 @@ type LocalProcessCluster struct { EnableSemiSync bool } +// Vttablet stores the properties needed to start a vttablet process +type Vttablet struct { + Type string + TabletUID int + HTTPPort int + GrpcPort int + MySQLPort int + Alias string + Cell string + + // background executable processes + MysqlctlProcess MysqlctlProcess + MysqlctldProcess MysqlctldProcess + VttabletProcess *VttabletProcess +} + // Keyspace : Cluster accepts keyspace to launch it type Keyspace struct { Name string diff --git a/go/test/endtoend/cluster/cluster_util.go b/go/test/endtoend/cluster/cluster_util.go index c140e44282e..4cc6404b54c 100644 --- a/go/test/endtoend/cluster/cluster_util.go +++ b/go/test/endtoend/cluster/cluster_util.go @@ -36,22 +36,6 @@ var ( tmClient = tmc.NewClient() ) -// Vttablet stores the properties needed to start a vttablet process -type Vttablet struct { - Type string - TabletUID int - HTTPPort int - GrpcPort int - MySQLPort int - Alias string - Cell string - - // background executable processes - MysqlctlProcess MysqlctlProcess - MysqlctldProcess MysqlctldProcess - VttabletProcess *VttabletProcess -} - // Restart restarts vttablet and mysql. func (tablet *Vttablet) Restart() error { if tablet.MysqlctlProcess.TabletUID|tablet.MysqlctldProcess.TabletUID == 0 { @@ -166,6 +150,9 @@ func getTablet(tabletGrpcPort int, hostname string) *tabletpb.Tablet { // NewConnParams creates ConnParams corresponds to given arguments. func NewConnParams(port int, password, socketPath, keyspace string) mysql.ConnParams { + if port != 0 { + socketPath = "" + } cp := mysql.ConnParams{ Uname: "vt_dba", Port: port, diff --git a/go/test/endtoend/mysqlctl/mysqlctl_test.go b/go/test/endtoend/mysqlctl/mysqlctl_test.go index c763e777482..abe7aa59622 100644 --- a/go/test/endtoend/mysqlctl/mysqlctl_test.go +++ b/go/test/endtoend/mysqlctl/mysqlctl_test.go @@ -58,7 +58,7 @@ func TestMain(m *testing.M) { initCluster([]string{"0"}, 2) - // Collect table paths and ports + // Collect tablet paths and ports tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets for _, tablet := range tablets { if tablet.Type == "master" { diff --git a/go/test/endtoend/mysqlctld/mysqlctld_test.go b/go/test/endtoend/mysqlctld/mysqlctld_test.go index 02c2889c9c4..d57ef1f5a59 100644 --- a/go/test/endtoend/mysqlctld/mysqlctld_test.go +++ b/go/test/endtoend/mysqlctld/mysqlctld_test.go @@ -59,7 +59,7 @@ func TestMain(m *testing.M) { return 1 } - // Collect table paths and ports + // Collect tablet paths and ports tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets for _, tablet := range tablets { if tablet.Type == "master" { diff --git a/test/config.json b/test/config.json index 7298e383e08..018a453729a 100644 --- a/test/config.json +++ b/test/config.json @@ -192,7 +192,7 @@ }, "backup_transform_mysqlctld": { "File": "backup_transform_mysqlctld_test.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backup/transformmysqlctld"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/transform/mysqlctld"], "Command": [], "Manual": false, "Shard": 21, From e17988d2569222de30ef2900c53b19838eb98a61 Mon Sep 17 00:00:00 2001 From: pradip parmar Date: Thu, 6 Feb 2020 15:42:56 +0530 Subject: [PATCH 50/50] mysqlctld: process hang issue resolved. Signed-off-by: pradip parmar --- go/test/endtoend/cluster/mysqlctld_process.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/cluster/mysqlctld_process.go b/go/test/endtoend/cluster/mysqlctld_process.go index aeb4ce6b3ba..855315fbe7d 100644 --- a/go/test/endtoend/cluster/mysqlctld_process.go +++ b/go/test/endtoend/cluster/mysqlctld_process.go @@ -118,7 +118,7 @@ func (mysqlctld *MysqlctldProcess) Start() error { } } - return fmt.Errorf("process '%s' timed out after 60s (err: %s)", mysqlctld.Name, <-mysqlctld.exit) + return fmt.Errorf("process '%s' timed out after 60s (err: %s)", mysqlctld.Name, mysqlctld.Stop()) } @@ -164,7 +164,7 @@ func MysqlCtldProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) // IsHealthy gives the health status of mysql. func (mysqlctld *MysqlctldProcess) IsHealthy() bool { socketFile := path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", mysqlctld.TabletUID), "/mysql.sock") - params := NewConnParams(mysqlctld.MySQLPort, mysqlctld.Password, socketFile, "") + params := NewConnParams(0, mysqlctld.Password, socketFile, "") _, err := mysql.Connect(context.Background(), ¶ms) return err == nil }