migrating mysqlctld python testcases to go

go/test/endtoend/backup/backupmysqlctld/backup_mysqlctld_test.go

@@ -0,0 +1,28 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package backupmysqlctld
+
+import (
+	"testing"
+
+	backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup"
+)
+
+// TestBackupMysqlctld - tests the backup using mysqlctld.
+func TestBackupMysqlctld(t *testing.T) {
+	backup.TestBackup(t, backup.Mysqlctld, "", 0)
+}

go/test/endtoend/backup/vtbackup/backup_only_test.go

@@ -237,7 +237,7 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS
 	// Erase mysql/tablet dir, then start tablet with restore enabled.
 
 	log.Info("restoring tablet %s", time.Now())
-	resetTabletDirectory(t, *tablet, true)
+	tablet.ValidateTabletRestart(t)
 
 	err := tablet.VttabletProcess.CreateDB(keyspaceName)
 	assert.Nil(t, err)
@@ -251,31 +251,6 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS
 	assert.Nil(t, err)
 }
 
-func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet, initMysql bool) {
-
-	extraArgs := []string{"-db-credentials-file", dbCredentialFile}
-	tablet.MysqlctlProcess.ExtraArgs = extraArgs
-
-	// Shutdown Mysql
-	err := tablet.MysqlctlProcess.Stop()
-	assert.Nil(t, err)
-	// Teardown Tablet
-	err = tablet.VttabletProcess.TearDown()
-	assert.Nil(t, err)
-
-	// Empty the dir
-	err = os.RemoveAll(tablet.VttabletProcess.Directory)
-	assert.Nil(t, err)
-
-	if initMysql {
-		// Init the Mysql
-		tablet.MysqlctlProcess.InitDBFile = newInitDBFile
-		err = tablet.MysqlctlProcess.Start()
-		assert.Nil(t, err)
-	}
-
-}
-
 func tearDown(t *testing.T, initMysql bool) {
 	// reset replication
 	promoteSlaveCommands := "STOP SLAVE; RESET SLAVE ALL; RESET MASTER;"
@@ -300,6 +275,6 @@ func tearDown(t *testing.T, initMysql bool) {
 		err := localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias)
 		assert.Nil(t, err)
 
-		resetTabletDirectory(t, tablet, initMysql)
+		tablet.ValidateTabletRestart(t)
 	}
 }

go/test/endtoend/backup/vtctlbackup/backup_test.go

@@ -16,18 +16,11 @@ limitations under the License.
 
 package vtctlbackup
 
-import "testing"
+import (
+	"testing"
+)
 
 // TestBackupMain - main tests backup using vtctl commands
 func TestBackupMain(t *testing.T) {
-	code, err := LaunchCluster(false, "", 0)
-	if err != nil {
-		t.Errorf("setup failed with status code %d", code)
-	}
-
-	// Run all the backup tests
-	TestBackup(t)
-
-	// Teardown the cluster
-	TearDownCluster()
+	TestBackup(t, Backup, "", 0)
 }

go/test/endtoend/backup/vtctlbackup/backup_utils.go

@@ -34,18 +34,23 @@ import (
 	"vitess.io/vitess/go/vt/proto/topodata"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"vitess.io/vitess/go/test/endtoend/cluster"
 )
 
+const (
+	ExtraBackup = iota
+	Backup
+	Mysqlctld
+)
+
 var (
 	master           *cluster.Vttablet
 	replica1         *cluster.Vttablet
 	replica2         *cluster.Vttablet
 	localCluster     *cluster.LocalProcessCluster
 	newInitDBFile    string
 	useXtrabackup    bool
-	xbStreamMode     string
-	xbStripes        int
 	cell             = cluster.DefaultCell
 	hostname         = "localhost"
 	keyspaceName     = "ks"
@@ -72,7 +77,8 @@ var (
 					  ) Engine=InnoDB`
 )
 
-func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error) {
+// LaunchCluster :
+func LaunchCluster(setupType int, streamMode string, stripes int) (int, error) {
 	localCluster = cluster.NewCluster(cell, hostname)
 
 	// Start topo server
@@ -101,16 +107,14 @@ func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error)
 	commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile)
 
 	// Update arguments for xtrabackup
-	if xtrabackup {
-		useXtrabackup = xtrabackup
-		xbStreamMode = streamMode
-		xbStripes = stripes
+	if setupType == ExtraBackup {
+		useXtrabackup = true
 
 		xtrabackupArgs := []string{
 			"-backup_engine_implementation", "xtrabackup",
-			fmt.Sprintf("-xtrabackup_stream_mode=%s", xbStreamMode),
+			fmt.Sprintf("-xtrabackup_stream_mode=%s", streamMode),
 			"-xtrabackup_user=vt_dba",
-			fmt.Sprintf("-xtrabackup_stripes=%d", xbStripes),
+			fmt.Sprintf("-xtrabackup_stripes=%d", stripes),
 			"-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword),
 		}
 
@@ -134,14 +138,27 @@ func LaunchCluster(xtrabackup bool, streamMode string, stripes int) (int, error)
 		tablet.VttabletProcess.SupportsBackup = true
 		tablet.VttabletProcess.EnableSemiSync = true
 
+		if setupType == Mysqlctld {
+			tablet.MysqlctldProcess = *cluster.MysqlCtldProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory)
+			tablet.MysqlctldProcess.InitDBFile = newInitDBFile
+			tablet.MysqlctldProcess.ExtraArgs = extraArgs
+			tablet.MysqlctldProcess.Password = tablet.VttabletProcess.DbPassword
+			if err := tablet.MysqlctldProcess.Start(); err != nil {
+				return 1, err
+			}
+			shard.Vttablets = append(shard.Vttablets, tablet)
+			continue
+		}
+
 		tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory)
 		tablet.MysqlctlProcess.InitDBFile = newInitDBFile
 		tablet.MysqlctlProcess.ExtraArgs = extraArgs
-		if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil {
+		proc, err := tablet.MysqlctlProcess.StartProcess()
+		if err != nil {
 			return 1, err
-		} else {
-			mysqlProcs = append(mysqlProcs, proc)
 		}
+		mysqlProcs = append(mysqlProcs, proc)
+
 		shard.Vttablets = append(shard.Vttablets, tablet)
 	}
 	for _, proc := range mysqlProcs {
@@ -179,35 +196,60 @@ func TearDownCluster() {
 	localCluster.Teardown()
 }
 
-func TestBackup(t *testing.T) {
-	// Run all the backup tests
-	t.Run("TestReplicaBackup", func(t *testing.T) {
-		vtctlBackup(t, "replica")
-	})
-
-	t.Run("TestRdonlyBackup", func(t *testing.T) {
-		vtctlBackup(t, "rdonly")
-	})
-
-	t.Run("TestMasterBackup", func(t *testing.T) {
-		masterBackup(t)
-	})
+func TestBackup(t *testing.T, setupType int, streamMode string, stripes int) {
+
+	testMethods := []struct {
+		name   string
+		method func(t *testing.T)
+	}{
+		{
+			name: "TestReplicaBackup",
+			method: func(t *testing.T) {
+				vtctlBackup(t, "replica")
+			},
+		}, //
+		{
+			name: "TestRdonlyBackup",
+			method: func(t *testing.T) {
+				vtctlBackup(t, "rdonly")
+			},
+		}, //
+		{
+			name:   "TestMasterBackup",
+			method: masterBackup,
+		}, //
+		{
+			name:   "TestMasterReplicaSameBackup",
+			method: masterReplicaSameBackup,
+		}, //
+		{
+			name:   "TestRestoreOldMasterByRestart",
+			method: restoreOldMasterByRestart,
+		}, //
+		{
+			name:   "TestRestoreOldMasterInPlace",
+			method: restoreOldMasterInPlace,
+		}, //
+		{
+			name:   "TestTerminatedRestore",
+			method: terminatedRestore,
+		}, //
+	}
 
-	t.Run("TestMasterReplicaSameBackup", func(t *testing.T) {
-		masterReplicaSameBackup(t)
-	})
+	// setup cluster for the testing
+	code, err := LaunchCluster(setupType, streamMode, stripes)
+	require.Nilf(t, err, "setup failed with status code %d", code)
 
-	t.Run("TestRestoreOldMasterByRestart", func(t *testing.T) {
-		restoreOldMasterByRestart(t)
-	})
+	// Teardown the cluster
+	defer TearDownCluster()
 
-	t.Run("TestRestoreOldMasterInPlace", func(t *testing.T) {
-		restoreOldMasterInPlace(t)
-	})
+	// Run all the backup tests
 
-	t.Run("TestTerminatedRestore", func(t *testing.T) {
-		terminatedRestore(t)
-	})
+	for _, test := range testMethods {
+		fmt.Printf("test %v started \n", test.name)
+		pass := t.Run(test.name, test.method)
+		fmt.Printf("test %v completed, isPass %v\n", test.name, pass)
+	}
 
 }
 
@@ -228,14 +270,12 @@ func masterBackup(t *testing.T) {
 	assert.NotNil(t, err)
 	assert.Contains(t, output, "type MASTER cannot take backup. if you really need to do this, rerun the backup command with -allow_master")
 
-	backups := listBackups(t)
-	assert.Equal(t, len(backups), 0)
+	localCluster.VerifyBackupCount(t, shardKsName, 0)
 
 	err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", "-allow_master=true", master.Alias)
 	assert.Nil(t, err)
 
-	backups = listBackups(t)
-	assert.Equal(t, len(backups), 1)
+	backups := localCluster.VerifyBackupCount(t, shardKsName, 1)
 	assert.Contains(t, backups[0], master.Alias)
 
 	_, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true)
@@ -383,14 +423,15 @@ func restartMasterReplica(t *testing.T) {
 	stopAllTablets()
 
 	// remove all backups
-	backups := listBackups(t)
-	for _, backup := range backups {
-		err := localCluster.VtctlclientProcess.ExecuteCommand("RemoveBackup", shardKsName, backup)
-		assert.Nil(t, err)
-	}
+	localCluster.RemoveAllBackups(t, shardKsName)
 	// start all tablet and mysql instances
 	var mysqlProcs []*exec.Cmd
-	for _, tablet := range []*cluster.Vttablet{master, replica1} {
+	for _, tablet := range []*cluster.Vttablet{master, replica1, replica2} {
+		if tablet.MysqlctldProcess.TabletUID > 0 {
+			err := tablet.MysqlctldProcess.Start()
+			require.Nilf(t, err, "error while starting mysqlctld, tabletUID %v", tablet.TabletUID)
+			continue
+		}
 		proc, _ := tablet.MysqlctlProcess.StartProcess()
 		mysqlProcs = append(mysqlProcs, proc)
 	}
@@ -413,6 +454,11 @@ func stopAllTablets() {
 	var mysqlProcs []*exec.Cmd
 	for _, tablet := range []*cluster.Vttablet{master, replica1, replica2} {
 		tablet.VttabletProcess.TearDown()
+		if tablet.MysqlctldProcess.TabletUID > 0 {
+			tablet.MysqlctldProcess.Stop()
+			localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias)
+			continue
+		}
 		proc, _ := tablet.MysqlctlProcess.StopProcess()
 		mysqlProcs = append(mysqlProcs, proc)
 		localCluster.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", tablet.Alias)
@@ -489,8 +535,7 @@ func vtctlBackup(t *testing.T, tabletType string) {
 	err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias)
 	assert.Nil(t, err)
 
-	backups := listBackups(t)
-	assert.Equal(t, len(backups), 1)
+	backups := localCluster.VerifyBackupCount(t, shardKsName, 1)
 
 	_, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true)
 	assert.Nil(t, err)
@@ -529,7 +574,7 @@ func verifyInitialReplication(t *testing.T) {
 // to restore a previous backup successfully regardless of this setting.
 func restoreWaitForBackup(t *testing.T, tabletType string) {
 	replica2.Type = tabletType
-	resetTabletDir(t, replica2)
+	replica2.ValidateTabletRestart(t)
 	replicaTabletArgs := commonTabletArg
 	replicaTabletArgs = append(replicaTabletArgs, "-backup_engine_implementation", "fake_implementation")
 	replicaTabletArgs = append(replicaTabletArgs, "-wait_for_backup_interval", "1s")
@@ -540,17 +585,6 @@ func restoreWaitForBackup(t *testing.T, tabletType string) {
 	assert.Nil(t, err)
 }
 
-func resetTabletDir(t *testing.T, tablet *cluster.Vttablet) {
-	err := cluster.ResetTabletDirectory(*tablet)
-	assert.Nil(t, err)
-}
-
-func listBackups(t *testing.T) []string {
-	output, err := localCluster.ListBackups(shardKsName)
-	assert.Nil(t, err)
-	return output
-}
-
 func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []string) {
 	// Remove the backup
 	for _, backup := range backups {
@@ -559,17 +593,14 @@ func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []st
 	}
 
 	// Now, there should not be no backup
-	backups = listBackups(t)
-	assert.Equal(t, len(backups), 0)
+	localCluster.VerifyBackupCount(t, shardKsName, 0)
 }
 
 func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string) {
-	err := tablet.VttabletProcess.TearDown()
-	assert.Nil(t, err)
 
-	resetTabletDir(t, tablet)
+	tablet.ValidateTabletRestart(t)
 	tablet.VttabletProcess.ServingStatus = ""
-	err = tablet.VttabletProcess.Setup()
+	err := tablet.VttabletProcess.Setup()
 	assert.Nil(t, err)
 	if status != "" {
 		err = tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{status}, 25*time.Second)
@@ -596,6 +627,7 @@ func terminateRestore(t *testing.T) {
 	stopRestoreMsg := "Copying file 10"
 	if useXtrabackup {
 		stopRestoreMsg = "Restore: Preparing"
+		useXtrabackup = false
 	}
 
 	args := append([]string{"-server", localCluster.VtctlclientProcess.Server, "-alsologtostderr"}, "RestoreFromBackup", master.Alias)