diff --git a/.github/workflows/cluster_endtoend_upgrade.yml b/.github/workflows/cluster_endtoend_upgrade.yml new file mode 100644 index 00000000000..185b9fbbe09 --- /dev/null +++ b/.github/workflows/cluster_endtoend_upgrade.yml @@ -0,0 +1,105 @@ +name: Cluster (upgrade) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (upgrade) + runs-on: ubuntu-latest + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Check out v8.0.0 + uses: actions/checkout@v2 + with: + ref: v8.0.0 + + - name: Get dependencies + run: | + # This prepares general purpose binary dependencies + # as well as v8.0.0 specific go modules + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Building v8.0.0 binaries + timeout-minutes: 10 + run: | + # We build v8.0.0 binaries and save them in a temporary location + source build.env + make build + mkdir -p /tmp/vitess-build-v8.0.0/ + cp -R bin /tmp/vitess-build-v8.0.0/ + + - name: Check out HEAD + uses: actions/checkout@v2 + + - name: Run cluster endtoend test v8.0.0 (create cluster) + timeout-minutes: 5 + run: | + # By checking out we deleted bin/ directory. We now restore our pre-built v8.0.0 binaries + cp -R /tmp/vitess-build-v8.0.0/bin . + # create the directory where we store test data; ensure it is empty: + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + source build.env + # We pass -skip-build so that we use the v8.0.0 binaries, not HEAD binaries + eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -shard 28 + + - name: Check out HEAD + uses: actions/checkout@v2 + + + - name: Building HEAD binaries + timeout-minutes: 10 + run: | + go mod download + + source build.env + make build + mkdir -p /tmp/vitess-build-head/ + cp -R bin /tmp/vitess-build-head/ + + - name: Run cluster endtoend test HEAD based on v8.0.0 data (upgrade path) + timeout-minutes: 5 + run: | + # /tmp/vtdataroot exists from previous test + + source build.env + # We built HEAD binaries manually in previous step and there's no need for the test to build. + eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -shard 28 + + - name: Run cluster endtoend test HEAD (create cluster) + timeout-minutes: 5 + run: | + # create the directory where we store test data; ensure it is empty: + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + # We still have the binaries from previous step. No need to build + eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -shard 28 + + + - name: Run cluster endtoend test v8.0.0 based on HEAD data (downgrade path) + timeout-minutes: 5 + run: | + # /tmp/vtdataroot exists from previous test + cp -R /tmp/vitess-build-v8.0.0/bin . + + source build.env + # We again built manually and there's no need for the test to build. + eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -shard 28 diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index 18049c90051..879b2ef8a0e 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -42,9 +42,12 @@ const ( ) var ( - keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders") - topoFlavor = flag.String("topo-flavor", "etcd2", "choose a topo server from etcd2, zk2 or consul") - isCoverage = flag.Bool("is-coverage", false, "whether coverage is required") + keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders") + topoFlavor = flag.String("topo-flavor", "etcd2", "choose a topo server from etcd2, zk2 or consul") + isCoverage = flag.Bool("is-coverage", false, "whether coverage is required") + forceVTDATAROOT = flag.String("force-vtdataroot", "", "force path for VTDATAROOT, which may already be populated") + forcePortStart = flag.Int("force-port-start", 0, "force assigning ports based on this seed") + forceBaseTabletUID = flag.Int("force-base-tablet-uid", 0, "force assigning tablet ports based on this seed") ) // LocalProcessCluster Testcases need to use this to iniate a cluster @@ -57,6 +60,7 @@ type LocalProcessCluster struct { TmpDirectory string OriginalVTDATAROOT string CurrentVTDATAROOT string + ReusingVTDATAROOT bool VtgateMySQLPort int VtgateGrpcPort int @@ -192,10 +196,12 @@ func (cluster *LocalProcessCluster) StartTopo() (err error) { } } - cluster.VtctlProcess = *VtctlProcessInstance(cluster.TopoProcess.Port, cluster.Hostname) - if err = cluster.VtctlProcess.AddCellInfo(cluster.Cell); err != nil { - log.Error(err) - return + if !cluster.ReusingVTDATAROOT { + cluster.VtctlProcess = *VtctlProcessInstance(cluster.TopoProcess.Port, cluster.Hostname) + if err = cluster.VtctlProcess.AddCellInfo(cluster.Cell); err != nil { + log.Error(err) + return + } } cluster.VtctldProcess = *VtctldProcessInstance(cluster.GetAndReservePort(), cluster.GetAndReservePort(), @@ -230,7 +236,9 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames } log.Infof("Starting keyspace: %v", keyspace.Name) - _ = cluster.VtctlProcess.CreateKeyspace(keyspace.Name) + if !cluster.ReusingVTDATAROOT { + _ = cluster.VtctlProcess.CreateKeyspace(keyspace.Name) + } var mysqlctlProcessList []*exec.Cmd for _, shardName := range shardNames { shard := &Shard{ @@ -256,7 +264,7 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames } // Start Mysqlctl process log.Infof("Starting mysqlctl for table uid %d, mysql port %d", tablet.TabletUID, tablet.MySQLPort) - tablet.MysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, cluster.TmpDirectory) + tablet.MysqlctlProcess = *MysqlCtlProcessInstanceOptionalInit(tablet.TabletUID, tablet.MySQLPort, cluster.TmpDirectory, !cluster.ReusingVTDATAROOT) proc, err := tablet.MysqlctlProcess.StartProcess() if err != nil { log.Errorf("error starting mysqlctl process: %v, %v", tablet.MysqlctldProcess, err) @@ -279,6 +287,9 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames cluster.VtTabletExtraArgs, cluster.EnableSemiSync) tablet.Alias = tablet.VttabletProcess.TabletPath + if cluster.ReusingVTDATAROOT { + tablet.VttabletProcess.ServingStatus = "SERVING" + } shard.Vttablets = append(shard.Vttablets, tablet) // Apply customizations for _, customizer := range customizers { @@ -298,9 +309,11 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames } } for _, tablet := range shard.Vttablets { - if _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("create database vt_%s", keyspace.Name), keyspace.Name, false); err != nil { - log.Errorf("error creating database for keyspace %v: %v", keyspace.Name, err) - return + if !cluster.ReusingVTDATAROOT { + if _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("create database vt_%s", keyspace.Name), keyspace.Name, false); err != nil { + log.Errorf("error creating database for keyspace %v: %v", keyspace.Name, err) + return + } } log.Infof("Starting vttablet for tablet uid %d, grpc port %d", tablet.TabletUID, tablet.GrpcPort) @@ -358,11 +371,13 @@ func (cluster *LocalProcessCluster) SetupCluster(keyspace *Keyspace, shards []Sh log.Infof("Starting keyspace: %v", keyspace.Name) - // Create Keyspace - err = cluster.VtctlProcess.CreateKeyspace(keyspace.Name) - if err != nil { - log.Error(err) - return + if !cluster.ReusingVTDATAROOT { + // Create Keyspace + err = cluster.VtctlProcess.CreateKeyspace(keyspace.Name) + if err != nil { + log.Error(err) + return + } } // Create shard @@ -441,8 +456,18 @@ func NewCluster(cell string, hostname string) *LocalProcessCluster { go cluster.CtrlCHandler() cluster.OriginalVTDATAROOT = os.Getenv("VTDATAROOT") cluster.CurrentVTDATAROOT = path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("vtroot_%d", cluster.GetAndReservePort())) - _ = createDirectory(cluster.CurrentVTDATAROOT, 0700) + if *forceVTDATAROOT != "" { + cluster.CurrentVTDATAROOT = *forceVTDATAROOT + } + if _, err := os.Stat(cluster.CurrentVTDATAROOT); err == nil { + // path/to/whatever exists + cluster.ReusingVTDATAROOT = true + } else { + _ = createDirectory(cluster.CurrentVTDATAROOT, 0700) + } _ = os.Setenv("VTDATAROOT", cluster.CurrentVTDATAROOT) + log.Infof("Created cluster on %s. ReusingVTDATAROOT=%v", cluster.CurrentVTDATAROOT, cluster.ReusingVTDATAROOT) + rand.Seed(time.Now().UTC().UnixNano()) return cluster } @@ -591,7 +616,11 @@ func (cluster *LocalProcessCluster) StartVtbackup(newInitDBFile string, initalBa // GetAndReservePort gives port for required process func (cluster *LocalProcessCluster) GetAndReservePort() int { if cluster.nextPortForProcess == 0 { - cluster.nextPortForProcess = getPort() + if *forcePortStart > 0 { + cluster.nextPortForProcess = *forcePortStart + } else { + cluster.nextPortForProcess = getPort() + } } for { cluster.nextPortForProcess = cluster.nextPortForProcess + 1 @@ -634,7 +663,11 @@ func getPort() int { // GetAndReserveTabletUID gives tablet uid func (cluster *LocalProcessCluster) GetAndReserveTabletUID() int { if cluster.BaseTabletUID == 0 { - cluster.BaseTabletUID = getRandomNumber(10000, 0) + if *forceBaseTabletUID > 0 { + cluster.BaseTabletUID = *forceBaseTabletUID + } else { + cluster.BaseTabletUID = getRandomNumber(10000, 0) + } } cluster.BaseTabletUID = cluster.BaseTabletUID + 1 return cluster.BaseTabletUID diff --git a/go/test/endtoend/cluster/mysqlctl_process.go b/go/test/endtoend/cluster/mysqlctl_process.go index 928ac6b3295..4954b3cb550 100644 --- a/go/test/endtoend/cluster/mysqlctl_process.go +++ b/go/test/endtoend/cluster/mysqlctl_process.go @@ -161,9 +161,9 @@ func (mysqlctl *MysqlctlProcess) CleanupFiles(tabletUID int) { os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/innodb", tabletUID))) } -// MysqlCtlProcessInstance returns a Mysqlctl handle for mysqlctl process +// MysqlCtlProcessInstanceOptionalInit returns a Mysqlctl handle for mysqlctl process // configured with the given Config. -func MysqlCtlProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) *MysqlctlProcess { +func MysqlCtlProcessInstanceOptionalInit(tabletUID int, mySQLPort int, tmpDirectory string, initMySQL bool) *MysqlctlProcess { mysqlctl := &MysqlctlProcess{ Name: "mysqlctl", Binary: "mysqlctl", @@ -172,11 +172,17 @@ func MysqlCtlProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) } mysqlctl.MySQLPort = mySQLPort mysqlctl.TabletUID = tabletUID - mysqlctl.InitMysql = true + mysqlctl.InitMysql = initMySQL mysqlctl.SecureTransport = false return mysqlctl } +// MysqlCtlProcessInstance returns a Mysqlctl handle for mysqlctl process +// configured with the given Config. +func MysqlCtlProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) *MysqlctlProcess { + return MysqlCtlProcessInstanceOptionalInit(tabletUID, mySQLPort, tmpDirectory, true) +} + // StartMySQL starts mysqlctl process func StartMySQL(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) error { tablet.MysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) diff --git a/go/test/endtoend/cluster/vtctlclient_process.go b/go/test/endtoend/cluster/vtctlclient_process.go index e94f83d76d1..1fd5d9ef00b 100644 --- a/go/test/endtoend/cluster/vtctlclient_process.go +++ b/go/test/endtoend/cluster/vtctlclient_process.go @@ -49,16 +49,20 @@ func (vtctlclient *VtctlClientProcess) InitShardMaster(Keyspace string, Shard st // ApplySchemaWithOutput applies SQL schema to the keyspace func (vtctlclient *VtctlClientProcess) ApplySchemaWithOutput(Keyspace string, SQL string, ddlStrategy string) (result string, err error) { - return vtctlclient.ExecuteCommandWithOutput( + args := []string{ "ApplySchema", "-sql", SQL, - "-ddl_strategy", ddlStrategy, - Keyspace) + } + if ddlStrategy != "" { + args = append(args, "-ddl_strategy", ddlStrategy) + } + args = append(args, Keyspace) + return vtctlclient.ExecuteCommandWithOutput(args...) } // ApplySchema applies SQL schema to the keyspace func (vtctlclient *VtctlClientProcess) ApplySchema(Keyspace string, SQL string) (err error) { - _, err = vtctlclient.ApplySchemaWithOutput(Keyspace, SQL, "") + _, err = vtctlclient.ApplySchemaWithOutput(Keyspace, SQL, "direct") return err } @@ -96,7 +100,7 @@ func (vtctlclient *VtctlClientProcess) OnlineDDLCancelMigration(Keyspace, uuid s ) } -// OnlineDDLCancelMigration cancels a given migration uuid +// OnlineDDLCancelAllMigrations cancels all migrations for a keyspace func (vtctlclient *VtctlClientProcess) OnlineDDLCancelAllMigrations(Keyspace string) (result string, err error) { return vtctlclient.ExecuteCommandWithOutput( "OnlineDDL", diff --git a/go/test/endtoend/versionupgrade/upgrade_test.go b/go/test/endtoend/versionupgrade/upgrade_test.go new file mode 100644 index 00000000000..f341bf792e0 --- /dev/null +++ b/go/test/endtoend/versionupgrade/upgrade_test.go @@ -0,0 +1,206 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + +ABOUT THIS TEST +=============== + +This test plays part in testing an upgrade path from a previous version/tag. It takes a GitHub workflow file to complete the functionality. +What's in this file is the setting up of a cluster, sharded and unsharded keyspace, creating and populating some tables, then testing retrieval of data. +The twist here is that you can run this test over pre-existing vtdataroot, which means this test can reuse existing etcd, existing tables, existing mysql, +in which case it will not attempt to create keyspaces/schemas/tables, nor will it populate table data. Instead, it will only check for retrieval of data. + +The game is to setup the cluster with a stable version (say `v8.0.0`), take it down (and preserve data), then setup a new cluster with a new version (namely the branch/PR head) and attempt to read the data. + +Both executions must force some settings so that both reuse same directories, ports, etc. An invocation will look like: +go test ./go/test/endtoend/versionupgrade80/upgrade80_test.go --keep-data -force-vtdataroot /tmp/vtdataroot/vtroot_10901 --force-port-start 11900 --force-base-tablet-uid 1190 + +*/ + +package versionupgrade + +import ( + "flag" + "fmt" + "os" + "path" + "testing" + + "vitess.io/vitess/go/mysql" + + "vitess.io/vitess/go/test/endtoend/cluster" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + hostname = "localhost" + keyspaceName = "ks" + cell = "zone1" + schemaChangeDirectory = "" + totalTableCount = 4 + createTable = ` + CREATE TABLE %s ( + id bigint(20) NOT NULL, + msg varchar(64), + PRIMARY KEY (id) + ) ENGINE=InnoDB; + ` + insertIntoTable = ` + INSERT INTO %s (id, msg) VALUES (17, 'abc'); + ` + selectFromTable = ` + SELECT id, msg FROM %s LIMIT 1; + ` +) + +// TestMain is the main entry point +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + flag.Parse() + + exitcode, err := func() (int, error) { + clusterInstance = cluster.NewCluster(cell, hostname) + schemaChangeDirectory = path.Join("/tmp", fmt.Sprintf("schema_change_dir_%d", clusterInstance.GetAndReserveTabletUID())) + defer os.RemoveAll(schemaChangeDirectory) + defer clusterInstance.Teardown() + + if _, err := os.Stat(schemaChangeDirectory); os.IsNotExist(err) { + _ = os.Mkdir(schemaChangeDirectory, 0700) + } + + clusterInstance.VtctldExtraArgs = []string{ + "-schema_change_dir", schemaChangeDirectory, + "-schema_change_controller", "local", + "-schema_change_check_interval", "1"} + + if err := clusterInstance.StartTopo(); err != nil { + return 1, err + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + } + + if err := clusterInstance.StartUnshardedKeyspace(*keyspace, 2, true); err != nil { + return 1, err + } + if err := clusterInstance.StartKeyspace(*keyspace, []string{"1"}, 1, false); err != nil { + return 1, err + } + + vtgateInstance := clusterInstance.NewVtgateInstance() + // set the gateway we want to use + vtgateInstance.GatewayImplementation = "tabletgateway" + // Start vtgate + if err := vtgateInstance.Setup(); err != nil { + return 1, err + } + // ensure it is torn down during cluster TearDown + clusterInstance.VtgateProcess = *vtgateInstance + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + + return m.Run(), nil + }() + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } else { + os.Exit(exitcode) + } + +} + +func TestShards(t *testing.T) { + defer cluster.PanicHandler(t) + assert.Equal(t, 2, len(clusterInstance.Keyspaces[0].Shards)) +} + +func TestDeploySchema(t *testing.T) { + defer cluster.PanicHandler(t) + + if clusterInstance.ReusingVTDATAROOT { + // we assume data is already deployed + return + } + // Create n tables, populate + for i := 0; i < totalTableCount; i++ { + tableName := fmt.Sprintf("vt_upgrade_test_%02d", i) + + { + sqlQuery := fmt.Sprintf(createTable, tableName) + _, err := clusterInstance.VtctlclientProcess.ApplySchemaWithOutput(keyspaceName, sqlQuery, "") + require.Nil(t, err) + } + for i := range clusterInstance.Keyspaces[0].Shards { + sqlQuery := fmt.Sprintf(insertIntoTable, tableName) + tablet := clusterInstance.Keyspaces[0].Shards[i].Vttablets[0] + _, err := tablet.VttabletProcess.QueryTablet(sqlQuery, keyspaceName, true) + require.Nil(t, err) + } + } + + checkTables(t, "", totalTableCount) +} + +func TestTablesExist(t *testing.T) { + defer cluster.PanicHandler(t) + + checkTables(t, "", totalTableCount) +} + +// checkTables checks the number of tables in the first two shards. +func checkTables(t *testing.T, showTableName string, expectCount int) { + for i := range clusterInstance.Keyspaces[0].Shards { + checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[i].Vttablets[0], showTableName, expectCount) + } +} + +// checkTablesCount checks the number of tables in the given tablet +func checkTablesCount(t *testing.T, tablet *cluster.Vttablet, showTableName string, expectCount int) { + query := fmt.Sprintf(`show tables like '%%%s%%';`, showTableName) + queryResult, err := tablet.VttabletProcess.QueryTablet(query, keyspaceName, true) + require.Nil(t, err) + assert.Equal(t, expectCount, len(queryResult.Rows)) +} + +// TestTablesData checks the data in tables +func TestTablesData(t *testing.T) { + // Create n tables, populate + for i := 0; i < totalTableCount; i++ { + tableName := fmt.Sprintf("vt_upgrade_test_%02d", i) + + for i := range clusterInstance.Keyspaces[0].Shards { + sqlQuery := fmt.Sprintf(selectFromTable, tableName) + tablet := clusterInstance.Keyspaces[0].Shards[i].Vttablets[0] + queryResult, err := tablet.VttabletProcess.QueryTablet(sqlQuery, keyspaceName, true) + require.Nil(t, err) + require.NotNil(t, queryResult) + row := queryResult.Named().Row() + require.NotNil(t, row) + require.Equal(t, int64(17), row.AsInt64("id", 0)) + require.Equal(t, "abc", row.AsString("msg", "")) + } + } +} diff --git a/test.go b/test.go index 3db180fc21c..bf355e39a72 100755 --- a/test.go +++ b/test.go @@ -90,6 +90,7 @@ var ( printLog = flag.Bool("print-log", false, "print the log of each failed test (or all tests if -log-pass) to the console") follow = flag.Bool("follow", false, "print test output as it runs, instead of waiting to see if it passes or fails") parallel = flag.Int("parallel", 1, "number of tests to run in parallel") + skipBuild = flag.Bool("skip-build", false, "skip running 'make build'. Assumes pre-existing binaries exist") remoteStats = flag.String("remote-stats", "", "url to send remote stats") ) @@ -380,6 +381,8 @@ func main() { log.Printf("Can't set permissions on temp dir %v: %v: %s", tmpDir, err, out) } vtRoot = tmpDir + } else if *skipBuild { + log.Printf("Skipping build...") } else { // Since we're sharing the working dir, do the build once for all tests. log.Printf("Running make build...") diff --git a/test/config.json b/test/config.json index 691ff4370a3..15b0fe79313 100644 --- a/test/config.json +++ b/test/config.json @@ -441,6 +441,15 @@ "site_test" ] }, + "upgrade": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/versionupgrade", "-keep-data", "-force-vtdataroot", "/tmp/vtdataroot/vtroot_10901", "-force-port-start", "11900", "-force-base-tablet-uid", "1190"], + "Command": [], + "Manual": false, + "Shard": 28, + "RetryMax": 0, + "Tags": [] + }, "vertical_split": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/sharding/verticalsplit"],