From 9d68d2c7d1ab820bac737d9d63465e4a7332c81b Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:23:26 +0300 Subject: [PATCH 01/14] vttablet: support init flags: --restore_to_timestamp, --restore_to_pos, behaving like --restore_from_backup but for PITR Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletmanager/restore.go | 34 +++++++++++++++++++++---- go/vt/vttablet/tabletmanager/tm_init.go | 31 +++++++++++++++++++--- go/vt/wrangler/testlib/backup_test.go | 12 ++++----- 3 files changed, 62 insertions(+), 15 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/restore.go b/go/vt/vttablet/tabletmanager/restore.go index ab1e32a5b5e..106ec2b5944 100644 --- a/go/vt/vttablet/tabletmanager/restore.go +++ b/go/vt/vttablet/tabletmanager/restore.go @@ -71,7 +71,18 @@ func registerRestoreFlags(fs *pflag.FlagSet) { } var ( - // Flags for PITR + // Flags for incremental restore (PITR) - new iteration + restoreToTimestampStr string + restoreToPos string +) + +func registerIncrementalRestoreFlags(fs *pflag.FlagSet) { + fs.StringVar(&restoreToTimestampStr, "restore_to_timestamp", restoreToTimestampStr, "(init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format (`2006-01-02T15:04:05Z07:00`)") + fs.StringVar(&restoreToPos, "restore_to_pos", restoreToPos, "(init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups") +} + +var ( + // Flags for PITR - old iteration binlogHost string binlogPort int binlogUser string @@ -99,6 +110,9 @@ func init() { servenv.OnParseFor("vtcombo", registerRestoreFlags) servenv.OnParseFor("vttablet", registerRestoreFlags) + servenv.OnParseFor("vtcombo", registerIncrementalRestoreFlags) + servenv.OnParseFor("vttablet", registerIncrementalRestoreFlags) + servenv.OnParseFor("vtcombo", registerPointInTimeRestoreFlags) servenv.OnParseFor("vttablet", registerPointInTimeRestoreFlags) @@ -110,7 +124,14 @@ func init() { // It will either work, fail gracefully, or return // an error in case of a non-recoverable error. // It takes the action lock so no RPC interferes. -func (tm *TabletManager) RestoreData(ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, deleteBeforeRestore bool, backupTime time.Time) error { +func (tm *TabletManager) RestoreData( + ctx context.Context, + logger logutil.Logger, + waitForBackupInterval time.Duration, + deleteBeforeRestore bool, + backupTime time.Time, + restoreToTimetamp time.Time, + restoreToPos string) error { if err := tm.lock(ctx); err != nil { return err } @@ -155,7 +176,9 @@ func (tm *TabletManager) RestoreData(ctx context.Context, logger logutil.Logger, startTime = time.Now() req := &tabletmanagerdatapb.RestoreFromBackupRequest{ - BackupTime: protoutil.TimeToProto(backupTime), + BackupTime: protoutil.TimeToProto(backupTime), + RestoreToPos: restoreToPos, + RestoreToTimestamp: protoutil.TimeToProto(restoreToTimetamp), } err = tm.restoreDataLocked(ctx, logger, waitForBackupInterval, deleteBeforeRestore, req) if err != nil { @@ -207,7 +230,8 @@ func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.L DryRun: request.DryRun, Stats: backupstats.RestoreStats(), } - if request.RestoreToPos != "" && !protoutil.TimeFromProto(request.RestoreToTimestamp).UTC().IsZero() { + restoreToTimestamp := protoutil.TimeFromProto(request.RestoreToTimestamp).UTC() + if request.RestoreToPos != "" && !restoreToTimestamp.IsZero() { return vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "--restore_to_pos and --restore_to_timestamp are mutually exclusive") } if request.RestoreToPos != "" { @@ -217,7 +241,7 @@ func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.L } params.RestoreToPos = pos } - if restoreToTimestamp := protoutil.TimeFromProto(request.RestoreToTimestamp).UTC(); !restoreToTimestamp.IsZero() { + if !restoreToTimestamp.IsZero() { // Restore to given timestamp params.RestoreToTimestamp = restoreToTimestamp } diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 14da1f9483d..783f0c2a445 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -766,8 +766,24 @@ func (tm *TabletManager) initTablet(ctx context.Context) error { func (tm *TabletManager) handleRestore(ctx context.Context) (bool, error) { // Sanity check for inconsistent flags - if tm.Cnf == nil && restoreFromBackup { - return false, fmt.Errorf("you cannot enable --restore_from_backup without a my.cnf file") + { + // mutually exclusive: + mutuallyExclusiveFlags := 0 + if restoreFromBackup { + mutuallyExclusiveFlags++ + } + if restoreToTimestampStr != "" { + mutuallyExclusiveFlags++ + } + if restoreToPos != "" { + mutuallyExclusiveFlags++ + } + if mutuallyExclusiveFlags > 1 { + return false, fmt.Errorf("You may only specify one out of --restore_from_backup, --restore_to_timestamp, --restore_to_pos") + } + if mutuallyExclusiveFlags > 0 && tm.Cnf == nil { + return false, fmt.Errorf("you cannot restore a tablet from backup without a my.cnf file") + } } // Restore in the background @@ -778,7 +794,6 @@ func (tm *TabletManager) handleRestore(ctx context.Context) (bool, error) { // Zero date will cause us to use the latest, which is the default backupTime := time.Time{} - // Or if a backup timestamp was specified then we use the last backup taken at or before that time if restoreFromBackupTsStr != "" { var err error @@ -788,9 +803,17 @@ func (tm *TabletManager) handleRestore(ctx context.Context) (bool, error) { } } + restoreToTimestamp := time.Time{} + if restoreToTimestampStr != "" { + var err error + restoreToTimestamp, err = mysqlctl.ParseRFC3339(restoreToTimestampStr) + if err != nil { + log.Exitf(fmt.Sprintf("RestoreFromBackup failed: unable to parse the --restore_to_timestamp value provided of '%s'. Error: %v", restoreToTimestampStr, err)) + } + } // restoreFromBackup will just be a regular action // (same as if it was triggered remotely) - if err := tm.RestoreData(ctx, logutil.NewConsoleLogger(), waitForBackupInterval, false /* deleteBeforeRestore */, backupTime); err != nil { + if err := tm.RestoreData(ctx, logutil.NewConsoleLogger(), waitForBackupInterval, false /* deleteBeforeRestore */, backupTime, restoreToTimestamp, restoreToPos); err != nil { log.Exitf("RestoreFromBackup failed: %v", err) } }() diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index 2cad68d71b3..787e4ce1946 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -263,7 +263,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { RelayLogInfoPath: path.Join(root, "relay-log.info"), } - err = destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* backupTime */) + err = destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* backupTime */, time.Time{} /* restoreToTimestamp */, "") if err != nil { return err } @@ -302,7 +302,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { primary.FakeMysqlDaemon.SetReplicationPositionPos = primary.FakeMysqlDaemon.CurrentPrimaryPosition // restore primary from latest backup - require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */), + require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, ""), "RestoreData failed") // tablet was created as PRIMARY, so it's baseTabletType is PRIMARY assert.Equal(t, topodatapb.TabletType_PRIMARY, primary.Tablet.Type) @@ -318,7 +318,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { } // Test restore with the backup timestamp - require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, backupTime), + require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, backupTime, time.Time{} /* restoreToTimestamp */, ""), "RestoreData with backup timestamp failed") assert.Equal(t, topodatapb.TabletType_PRIMARY, primary.Tablet.Type) assert.False(t, primary.FakeMysqlDaemon.Replicating) @@ -519,7 +519,7 @@ func TestBackupRestoreLagged(t *testing.T) { errCh = make(chan error, 1) go func(ctx context.Context, tablet *FakeTablet) { - errCh <- tablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */) + errCh <- tablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "") }(ctx, destTablet) timer = time.NewTicker(1 * time.Second) @@ -713,7 +713,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) { // set a short timeout so that we don't have to wait 30 seconds topo.RemoteOperationTimeout = 2 * time.Second // Restore should still succeed - require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */)) + require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "")) // verify the full status require.NoError(t, destTablet.FakeMysqlDaemon.CheckSuperQueryList(), "destTablet.FakeMysqlDaemon.CheckSuperQueryList failed") assert.True(t, destTablet.FakeMysqlDaemon.Replicating) @@ -867,7 +867,7 @@ func TestDisableActiveReparents(t *testing.T) { RelayLogInfoPath: path.Join(root, "relay-log.info"), } - require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */)) + require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "")) // verify the full status require.NoError(t, destTablet.FakeMysqlDaemon.CheckSuperQueryList(), "destTablet.FakeMysqlDaemon.CheckSuperQueryList failed") assert.False(t, destTablet.FakeMysqlDaemon.Replicating) From 813b7d76e8211161562a1b93466bd46827cb1a1e Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:25:03 +0300 Subject: [PATCH 02/14] adding a 'spare' tablet, unstarted, in backup_utils Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../backup/vtctlbackup/backup_utils.go | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 35edab98928..d2eb8079e52 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -61,6 +61,7 @@ var ( primary *cluster.Vttablet replica1 *cluster.Vttablet replica2 *cluster.Vttablet + replica3 *cluster.Vttablet localCluster *cluster.LocalProcessCluster newInitDBFile string useXtrabackup bool @@ -170,9 +171,10 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp 0: "primary", 1: "replica", 2: "rdonly", + 3: "spare", } - for i := 0; i < 3; i++ { - tabletType := tabletTypes[i] + + createTablet := func(tabletType string) error { tablet := localCluster.NewVttabletInstance(tabletType, 0, cell) tablet.VttabletProcess = localCluster.VtprocessInstanceFromVttablet(tablet, shard.Name, keyspaceName) tablet.VttabletProcess.DbPassword = dbPassword @@ -182,33 +184,40 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp if setupType == Mysqlctld { mysqlctldProcess, err := cluster.MysqlCtldProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) if err != nil { - return 1, err + return err } tablet.MysqlctldProcess = *mysqlctldProcess tablet.MysqlctldProcess.InitDBFile = newInitDBFile tablet.MysqlctldProcess.ExtraArgs = extraArgs tablet.MysqlctldProcess.Password = tablet.VttabletProcess.DbPassword if err := tablet.MysqlctldProcess.Start(); err != nil { - return 1, err + return err } shard.Vttablets = append(shard.Vttablets, tablet) - continue + return nil } mysqlctlProcess, err := cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) if err != nil { - return 1, err + return err } tablet.MysqlctlProcess = *mysqlctlProcess tablet.MysqlctlProcess.InitDBFile = newInitDBFile tablet.MysqlctlProcess.ExtraArgs = extraArgs proc, err := tablet.MysqlctlProcess.StartProcess() if err != nil { - return 1, err + return err } mysqlProcs = append(mysqlProcs, proc) shard.Vttablets = append(shard.Vttablets, tablet) + return nil + } + for i := 0; i < 3; i++ { + tabletType := tabletTypes[i] + if err := createTablet(tabletType); err != nil { + return 1, err + } } for _, proc := range mysqlProcs { if err := proc.Wait(); err != nil { @@ -218,6 +227,7 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp primary = shard.Vttablets[0] replica1 = shard.Vttablets[1] replica2 = shard.Vttablets[2] + replica3 = shard.Vttablets[3] if err := localCluster.VtctlclientProcess.InitTablet(primary, cell, keyspaceName, hostname, shard.Name); err != nil { return 1, err @@ -228,13 +238,16 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp if err := localCluster.VtctlclientProcess.InitTablet(replica2, cell, keyspaceName, hostname, shard.Name); err != nil { return 1, err } + if err := localCluster.VtctlclientProcess.InitTablet(replica3, cell, keyspaceName, hostname, shard.Name); err != nil { + return 1, err + } vtctldClientProcess := cluster.VtctldClientProcessInstance("localhost", localCluster.VtctldProcess.GrpcPort, localCluster.TmpDirectory) _, err = vtctldClientProcess.ExecuteCommandWithOutput("SetKeyspaceDurabilityPolicy", keyspaceName, "--durability-policy=semi_sync") if err != nil { return 1, err } - for _, tablet := range []*cluster.Vttablet{primary, replica1, replica2} { + for _, tablet := range []*cluster.Vttablet{primary, replica1, replica2} { // we don't start replica3 yet if err := tablet.VttabletProcess.Setup(); err != nil { return 1, err } From e832cc29d145b99b6a4740762e8e6edf9a56e316 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:28:37 +0300 Subject: [PATCH 03/14] iterate all four tablets Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/backup/vtctlbackup/backup_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index d2eb8079e52..775e85f3bd3 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -213,7 +213,7 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp shard.Vttablets = append(shard.Vttablets, tablet) return nil } - for i := 0; i < 3; i++ { + for i := 0; i < 4; i++ { tabletType := tabletTypes[i] if err := createTablet(tabletType); err != nil { return 1, err From c11af8a2a34a3e75349ee54a0d0499018a28160b Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 12 Sep 2023 17:33:08 +0300 Subject: [PATCH 04/14] --restore_to_timestamp requires --restore_from_backup ; same for __restore_to_pos Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletmanager/tm_init.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 783f0c2a445..d96e9116830 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -766,24 +766,11 @@ func (tm *TabletManager) initTablet(ctx context.Context) error { func (tm *TabletManager) handleRestore(ctx context.Context) (bool, error) { // Sanity check for inconsistent flags - { - // mutually exclusive: - mutuallyExclusiveFlags := 0 - if restoreFromBackup { - mutuallyExclusiveFlags++ - } - if restoreToTimestampStr != "" { - mutuallyExclusiveFlags++ - } - if restoreToPos != "" { - mutuallyExclusiveFlags++ - } - if mutuallyExclusiveFlags > 1 { - return false, fmt.Errorf("You may only specify one out of --restore_from_backup, --restore_to_timestamp, --restore_to_pos") - } - if mutuallyExclusiveFlags > 0 && tm.Cnf == nil { - return false, fmt.Errorf("you cannot restore a tablet from backup without a my.cnf file") - } + if tm.Cnf == nil && restoreFromBackup { + return false, fmt.Errorf("you cannot enable --restore_from_backup without a my.cnf file") + } + if restoreToTimestampStr != "" && restoreToPos != "" { + return false, fmt.Errorf("--restore_to_timestamp and --restore_to_pos are mutually exclusive") } // Restore in the background From 1ccd02b4c6c43e5af3b6677664772fc2f42a3e37 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 12 Sep 2023 17:34:40 +0300 Subject: [PATCH 05/14] backup_pitr tests: reserve a third replica, only initialize and start it after backups have been taken, bootstrap it via --restore_from_backup --restore_to_timestamp, or --restore_from_backup --restore_to_pos. Validate that it restores into expected point in time with expected number of rows, and that it ends in DRAINED type Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../endtoend/backup/pitr/backup_pitr_test.go | 20 +++++-- .../backup/vtctlbackup/backup_utils.go | 19 +++++-- .../backup/vtctlbackup/pitr_test_framework.go | 53 +++++++++++++++++++ 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/go/test/endtoend/backup/pitr/backup_pitr_test.go b/go/test/endtoend/backup/pitr/backup_pitr_test.go index a1b29ef47dd..24767e5db02 100644 --- a/go/test/endtoend/backup/pitr/backup_pitr_test.go +++ b/go/test/endtoend/backup/pitr/backup_pitr_test.go @@ -22,7 +22,21 @@ import ( backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" ) -// TestIncrementalBackupAndRestoreToPos +// TestIncrementalBackupAndRestoreToPos - tests incremental backups and restores. +// The general outline of the test: +// - Generate some schema with data +// - Take a full backup +// - Proceed to take a series of inremental backups. In between, inject data (insert rows), and keep record +// of which data (number of rows) is present in each backup, and at which position. +// - Expect backups success/failure per scenario +// - Next up, we start testing restores. Randomly pick recorded positions and restore to those points in time. +// - In each restore, excpect to find the data (number of rows) recorded for said position +// - Some restores should fail because the position exceeds the last binlog +// - Do so for all recorded positions. +// - Then, a 2nd round where some backups are purged -- this tests to see that we're still able to find a restore path +// (of course we only delete backups that still leave us with valid restore paths). +// - Last, create a new tablet with --restore_from_backup --restore_to_pos and see that it bootstraps with restored data +// and that it ends up in DRAINED type func TestIncrementalBackupAndRestoreToPos(t *testing.T) { tcase := &backup.PITRTestCase{ Name: "BuiltinBackup", @@ -45,8 +59,8 @@ func TestIncrementalBackupAndRestoreToPos(t *testing.T) { // - Do so for all recorded tiemstamps. // - Then, a 2nd round where some backups are purged -- this tests to see that we're still able to find a restore path // (of course we only delete backups that still leave us with valid restore paths). -// -// All of the above is done for BuiltinBackup, XtraBackup, Mysqlctld (which is technically builtin) +// - Last, create a new tablet with --restore_from_backup --restore_to_timestamp and see that it bootstraps with restored data +// and that it ends up in DRAINED type func TestIncrementalBackupAndRestoreToTimestamp(t *testing.T) { tcase := &backup.PITRTestCase{ Name: "BuiltinBackup", diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 775e85f3bd3..e3b1c6ac1a9 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -91,6 +91,7 @@ var ( primary key (id) ) Engine=InnoDB ` + SetupReplica3Tablet func(extraArgs []string) (*cluster.Vttablet, error) ) type CompressionDetails struct { @@ -238,9 +239,6 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp if err := localCluster.VtctlclientProcess.InitTablet(replica2, cell, keyspaceName, hostname, shard.Name); err != nil { return 1, err } - if err := localCluster.VtctlclientProcess.InitTablet(replica3, cell, keyspaceName, hostname, shard.Name); err != nil { - return 1, err - } vtctldClientProcess := cluster.VtctldClientProcessInstance("localhost", localCluster.VtctldProcess.GrpcPort, localCluster.TmpDirectory) _, err = vtctldClientProcess.ExecuteCommandWithOutput("SetKeyspaceDurabilityPolicy", keyspaceName, "--durability-policy=semi_sync") if err != nil { @@ -253,6 +251,17 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp } } + SetupReplica3Tablet = func(extraArgs []string) (*cluster.Vttablet, error) { + replica3.VttabletProcess.ExtraArgs = append(replica3.VttabletProcess.ExtraArgs, extraArgs...) + if err := localCluster.VtctlclientProcess.InitTablet(replica3, cell, keyspaceName, hostname, shard.Name); err != nil { + return replica3, err + } + if err := replica3.VttabletProcess.Setup(); err != nil { + return replica3, err + } + return replica3, nil + } + if err := localCluster.VtctlclientProcess.InitShardPrimary(keyspaceName, shard.Name, cell, primary.TabletUID); err != nil { return 1, err } @@ -1153,6 +1162,8 @@ func getReplica(t *testing.T, replicaIndex int) *cluster.Vttablet { return replica1 case 1: return replica2 + case 2: + return replica3 default: assert.Failf(t, "invalid replica index", "index=%d", replicaIndex) return nil @@ -1303,6 +1314,7 @@ func TestReplicaRestoreToPos(t *testing.T, replicaIndex int, restoreToPos replic } require.NoErrorf(t, err, "output: %v", output) verifyTabletRestoreStats(t, replica.VttabletProcess.GetVars()) + checkTabletType(t, replica1.Alias, topodata.TabletType_DRAINED) } func TestReplicaRestoreToTimestamp(t *testing.T, restoreToTimestamp time.Time, expectError string) { @@ -1316,6 +1328,7 @@ func TestReplicaRestoreToTimestamp(t *testing.T, restoreToTimestamp time.Time, e } require.NoErrorf(t, err, "output: %v", output) verifyTabletRestoreStats(t, replica1.VttabletProcess.GetVars()) + checkTabletType(t, replica1.Alias, topodata.TabletType_DRAINED) } func verifyTabletBackupStats(t *testing.T, vars map[string]any) { diff --git a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go index 2468940b641..89bf61840e7 100644 --- a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go +++ b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go @@ -34,6 +34,7 @@ import ( var ( gracefulPostBackupDuration = 10 * time.Millisecond + backupTimeoutDuration = 3 * time.Minute ) const ( @@ -225,6 +226,7 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) }) } + sampleTestedBackupPos := "" testRestores := func(t *testing.T) { for _, r := range rand.Perm(len(backupPositions)) { pos := backupPositions[r] @@ -237,6 +239,9 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) count, ok := rowsPerPosition[pos] require.True(t, ok) assert.Equalf(t, count, len(msgs), "messages: %v", msgs) + if sampleTestedBackupPos == "" { + sampleTestedBackupPos = pos + } }) } } @@ -252,6 +257,27 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) t.Run("PITR-2", func(t *testing.T) { testRestores(t) }) + // Test that we can create a new tablet with --restore_from_backup --restore_to_pos and that it bootstraps + // via PITR and ends up in DRAINED type. + t.Run("init tablet PITR", func(t *testing.T) { + require.NotEmpty(t, sampleTestedBackupPos) + + var tablet *cluster.Vttablet + + t.Run(fmt.Sprintf("init from backup pos %s", sampleTestedBackupPos), func(t *testing.T) { + tablet, err = SetupReplica3Tablet([]string{"--restore_to_pos", sampleTestedBackupPos}) + assert.NoError(t, err) + }) + t.Run("wait for drained", func(t *testing.T) { + err = tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{"drained"}, backupTimeoutDuration) + assert.NoError(t, err) + }) + t.Run(fmt.Sprintf("validate %d rows", rowsPerPosition[sampleTestedBackupPos]), func(t *testing.T) { + require.NotZero(t, rowsPerPosition[sampleTestedBackupPos]) + msgs := ReadRowsFromReplica(t, 2) + assert.Equal(t, rowsPerPosition[sampleTestedBackupPos], len(msgs)) + }) + }) }) } @@ -415,6 +441,7 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes }) } + sampleTestedBackupIndex := -1 testRestores := func(t *testing.T) { numFailedRestores := 0 numSuccessfulRestores := 0 @@ -433,6 +460,9 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes msgs := ReadRowsFromReplica(t, 0) assert.Equalf(t, testedBackup.rows, len(msgs), "messages: %v", msgs) numSuccessfulRestores++ + if sampleTestedBackupIndex < 0 { + sampleTestedBackupIndex = backupIndex + } } else { numFailedRestores++ } @@ -454,6 +484,29 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes t.Run("PITR-2", func(t *testing.T) { testRestores(t) }) + // Test that we can create a new tablet with --restore_from_backup --restore_to_timestamp and that it bootstraps + // via PITR and ends up in DRAINED type. + t.Run("init tablet PITR", func(t *testing.T) { + require.GreaterOrEqual(t, sampleTestedBackupIndex, 0) + sampleTestedBackup := testedBackups[sampleTestedBackupIndex] + restoreToTimestampArg := mysqlctl.FormatRFC3339(sampleTestedBackup.postTimestamp) + + var tablet *cluster.Vttablet + + t.Run(fmt.Sprintf("init from backup num %d", sampleTestedBackupIndex), func(t *testing.T) { + tablet, err = SetupReplica3Tablet([]string{"--restore_to_timestamp", restoreToTimestampArg}) + assert.NoError(t, err) + }) + t.Run("wait for drained", func(t *testing.T) { + err = tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{"drained"}, backupTimeoutDuration) + assert.NoError(t, err) + }) + t.Run(fmt.Sprintf("validate %d rows", sampleTestedBackup.rows), func(t *testing.T) { + require.NotZero(t, sampleTestedBackup.rows) + msgs := ReadRowsFromReplica(t, 2) + assert.Equal(t, sampleTestedBackup.rows, len(msgs)) + }) + }) }) } From 2b20ce72e7799440a871d5a686ac24b5cdccbd15 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 13 Sep 2023 10:36:23 +0300 Subject: [PATCH 06/14] fixed vttablet flags test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/flags/endtoend/vttablet.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 9b42ebb5644..dac25ec7852 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -256,6 +256,8 @@ Usage of vttablet: --restore_concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) --restore_from_backup (init restore parameter) will check BackupStorage for a recent backup at startup and start there --restore_from_backup_ts string (init restore parameter) if set, restore the latest backup taken at or before this timestamp. Example: '2021-04-29.133050' + --restore_to_pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups + --restore_to_timestamp 2006-01-02T15:04:05Z07:00 (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format (2006-01-02T15:04:05Z07:00) --retain_online_ddl_tables duration How long should vttablet keep an old migrated table before purging it (default 24h0m0s) --s2a_enable_appengine_dialer If true, opportunistically use AppEngine-specific dialer to call S2A. --s2a_timeout duration Timeout enforced on the connection to the S2A service for handshake. (default 3s) From 31b67250242bc6ea2baec689b12ce6a54939356e Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:42:10 +0300 Subject: [PATCH 07/14] No need for InitTablet Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/backup/vtctlbackup/backup_utils.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index e3b1c6ac1a9..1ca56db68c2 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -253,9 +253,6 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp SetupReplica3Tablet = func(extraArgs []string) (*cluster.Vttablet, error) { replica3.VttabletProcess.ExtraArgs = append(replica3.VttabletProcess.ExtraArgs, extraArgs...) - if err := localCluster.VtctlclientProcess.InitTablet(replica3, cell, keyspaceName, hostname, shard.Name); err != nil { - return replica3, err - } if err := replica3.VttabletProcess.Setup(); err != nil { return replica3, err } From 029efbc3dd54799868c74383f92c310a21429319 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 21 Sep 2023 18:40:55 +0300 Subject: [PATCH 08/14] fix '--restore_to_timestamp' comment Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/flags/endtoend/vttablet.txt | 2 +- go/vt/vttablet/tabletmanager/restore.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index dac25ec7852..ce4231d5549 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -257,7 +257,7 @@ Usage of vttablet: --restore_from_backup (init restore parameter) will check BackupStorage for a recent backup at startup and start there --restore_from_backup_ts string (init restore parameter) if set, restore the latest backup taken at or before this timestamp. Example: '2021-04-29.133050' --restore_to_pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups - --restore_to_timestamp 2006-01-02T15:04:05Z07:00 (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format (2006-01-02T15:04:05Z07:00) + --restore_to_timestamp string (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00' --retain_online_ddl_tables duration How long should vttablet keep an old migrated table before purging it (default 24h0m0s) --s2a_enable_appengine_dialer If true, opportunistically use AppEngine-specific dialer to call S2A. --s2a_timeout duration Timeout enforced on the connection to the S2A service for handshake. (default 3s) diff --git a/go/vt/vttablet/tabletmanager/restore.go b/go/vt/vttablet/tabletmanager/restore.go index 106ec2b5944..ba120f0ddb4 100644 --- a/go/vt/vttablet/tabletmanager/restore.go +++ b/go/vt/vttablet/tabletmanager/restore.go @@ -77,7 +77,7 @@ var ( ) func registerIncrementalRestoreFlags(fs *pflag.FlagSet) { - fs.StringVar(&restoreToTimestampStr, "restore_to_timestamp", restoreToTimestampStr, "(init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format (`2006-01-02T15:04:05Z07:00`)") + fs.StringVar(&restoreToTimestampStr, "restore_to_timestamp", restoreToTimestampStr, "(init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00'") fs.StringVar(&restoreToPos, "restore_to_pos", restoreToPos, "(init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups") } From 345a2522170a2c01c174363c3bb7816f45b83d3a Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 27 Sep 2023 08:42:28 +0300 Subject: [PATCH 09/14] vtcombo flags docs Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/flags/endtoend/vtcombo.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index ffa2b84970a..427ecc97cf4 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -303,6 +303,8 @@ Flags: --restore_concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) --restore_from_backup (init restore parameter) will check BackupStorage for a recent backup at startup and start there --restore_from_backup_ts string (init restore parameter) if set, restore the latest backup taken at or before this timestamp. Example: '2021-04-29.133050' + --restore_to_pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups + --restore_to_timestamp string (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00' --retain_online_ddl_tables duration How long should vttablet keep an old migrated table before purging it (default 24h0m0s) --sanitize_log_messages Remove potentially sensitive information in tablet INFO, WARNING, and ERROR log messages such as query parameters. --schema-change-reload-timeout duration query server schema change reload timeout, this is how long to wait for the signaled schema reload operation to complete before giving up (default 30s) From f376b60c756d3505b80564be2b99fb6bcf9ba394 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 27 Sep 2023 15:11:25 +0300 Subject: [PATCH 10/14] dashes, not underscores Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/flags/endtoend/vtcombo.txt | 4 ++-- go/flags/endtoend/vttablet.txt | 4 ++-- go/test/endtoend/backup/pitr/backup_pitr_test.go | 4 ++-- go/test/endtoend/backup/vtctlbackup/backup_utils.go | 2 +- .../endtoend/backup/vtctlbackup/pitr_test_framework.go | 8 ++++---- go/vt/vttablet/tabletmanager/restore.go | 8 ++++---- go/vt/vttablet/tabletmanager/tm_init.go | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 427ecc97cf4..b5a0f0b03e9 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -303,8 +303,8 @@ Flags: --restore_concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) --restore_from_backup (init restore parameter) will check BackupStorage for a recent backup at startup and start there --restore_from_backup_ts string (init restore parameter) if set, restore the latest backup taken at or before this timestamp. Example: '2021-04-29.133050' - --restore_to_pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups - --restore_to_timestamp string (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00' + --restore-to-pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups + --restore-to-timestamp string (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00' --retain_online_ddl_tables duration How long should vttablet keep an old migrated table before purging it (default 24h0m0s) --sanitize_log_messages Remove potentially sensitive information in tablet INFO, WARNING, and ERROR log messages such as query parameters. --schema-change-reload-timeout duration query server schema change reload timeout, this is how long to wait for the signaled schema reload operation to complete before giving up (default 30s) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index f9192216ca4..a90f93941c6 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -297,8 +297,8 @@ Flags: --restore_concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) --restore_from_backup (init restore parameter) will check BackupStorage for a recent backup at startup and start there --restore_from_backup_ts string (init restore parameter) if set, restore the latest backup taken at or before this timestamp. Example: '2021-04-29.133050' - --restore_to_pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups - --restore_to_timestamp string (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00' + --restore-to-pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups + --restore-to-timestamp string (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00' --retain_online_ddl_tables duration How long should vttablet keep an old migrated table before purging it (default 24h0m0s) --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). --s3_backup_aws_region string AWS region to use. (default "us-east-1") diff --git a/go/test/endtoend/backup/pitr/backup_pitr_test.go b/go/test/endtoend/backup/pitr/backup_pitr_test.go index 24767e5db02..fcf8e9490e8 100644 --- a/go/test/endtoend/backup/pitr/backup_pitr_test.go +++ b/go/test/endtoend/backup/pitr/backup_pitr_test.go @@ -35,7 +35,7 @@ import ( // - Do so for all recorded positions. // - Then, a 2nd round where some backups are purged -- this tests to see that we're still able to find a restore path // (of course we only delete backups that still leave us with valid restore paths). -// - Last, create a new tablet with --restore_from_backup --restore_to_pos and see that it bootstraps with restored data +// - Last, create a new tablet with --restore_from_backup --restore-to-pos and see that it bootstraps with restored data // and that it ends up in DRAINED type func TestIncrementalBackupAndRestoreToPos(t *testing.T) { tcase := &backup.PITRTestCase{ @@ -59,7 +59,7 @@ func TestIncrementalBackupAndRestoreToPos(t *testing.T) { // - Do so for all recorded tiemstamps. // - Then, a 2nd round where some backups are purged -- this tests to see that we're still able to find a restore path // (of course we only delete backups that still leave us with valid restore paths). -// - Last, create a new tablet with --restore_from_backup --restore_to_timestamp and see that it bootstraps with restored data +// - Last, create a new tablet with --restore_from_backup --restore-to-timestamp and see that it bootstraps with restored data // and that it ends up in DRAINED type func TestIncrementalBackupAndRestoreToTimestamp(t *testing.T) { tcase := &backup.PITRTestCase{ diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 1ca56db68c2..e8adbd1af18 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -1303,7 +1303,7 @@ func TestReplicaRestoreToPos(t *testing.T, replicaIndex int, restoreToPos replic require.False(t, restoreToPos.IsZero()) restoreToPosArg := replication.EncodePosition(restoreToPos) - output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--", "--restore_to_pos", restoreToPosArg, replica.Alias) + output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--", "--restore-to-pos", restoreToPosArg, replica.Alias) if expectError != "" { require.Errorf(t, err, "expected: %v", expectError) require.Contains(t, output, expectError) diff --git a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go index 89bf61840e7..8b9014e7f8c 100644 --- a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go +++ b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go @@ -257,7 +257,7 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) t.Run("PITR-2", func(t *testing.T) { testRestores(t) }) - // Test that we can create a new tablet with --restore_from_backup --restore_to_pos and that it bootstraps + // Test that we can create a new tablet with --restore_from_backup --restore-to-pos and that it bootstraps // via PITR and ends up in DRAINED type. t.Run("init tablet PITR", func(t *testing.T) { require.NotEmpty(t, sampleTestedBackupPos) @@ -265,7 +265,7 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) var tablet *cluster.Vttablet t.Run(fmt.Sprintf("init from backup pos %s", sampleTestedBackupPos), func(t *testing.T) { - tablet, err = SetupReplica3Tablet([]string{"--restore_to_pos", sampleTestedBackupPos}) + tablet, err = SetupReplica3Tablet([]string{"--restore-to-pos", sampleTestedBackupPos}) assert.NoError(t, err) }) t.Run("wait for drained", func(t *testing.T) { @@ -484,7 +484,7 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes t.Run("PITR-2", func(t *testing.T) { testRestores(t) }) - // Test that we can create a new tablet with --restore_from_backup --restore_to_timestamp and that it bootstraps + // Test that we can create a new tablet with --restore_from_backup --restore-to-timestamp and that it bootstraps // via PITR and ends up in DRAINED type. t.Run("init tablet PITR", func(t *testing.T) { require.GreaterOrEqual(t, sampleTestedBackupIndex, 0) @@ -494,7 +494,7 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes var tablet *cluster.Vttablet t.Run(fmt.Sprintf("init from backup num %d", sampleTestedBackupIndex), func(t *testing.T) { - tablet, err = SetupReplica3Tablet([]string{"--restore_to_timestamp", restoreToTimestampArg}) + tablet, err = SetupReplica3Tablet([]string{"--restore-to-timestamp", restoreToTimestampArg}) assert.NoError(t, err) }) t.Run("wait for drained", func(t *testing.T) { diff --git a/go/vt/vttablet/tabletmanager/restore.go b/go/vt/vttablet/tabletmanager/restore.go index ba120f0ddb4..4512b546f2c 100644 --- a/go/vt/vttablet/tabletmanager/restore.go +++ b/go/vt/vttablet/tabletmanager/restore.go @@ -77,8 +77,8 @@ var ( ) func registerIncrementalRestoreFlags(fs *pflag.FlagSet) { - fs.StringVar(&restoreToTimestampStr, "restore_to_timestamp", restoreToTimestampStr, "(init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00'") - fs.StringVar(&restoreToPos, "restore_to_pos", restoreToPos, "(init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups") + fs.StringVar(&restoreToTimestampStr, "restore-to-timestamp", restoreToTimestampStr, "(init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00'") + fs.StringVar(&restoreToPos, "restore-to-pos", restoreToPos, "(init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups") } var ( @@ -232,12 +232,12 @@ func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.L } restoreToTimestamp := protoutil.TimeFromProto(request.RestoreToTimestamp).UTC() if request.RestoreToPos != "" && !restoreToTimestamp.IsZero() { - return vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "--restore_to_pos and --restore_to_timestamp are mutually exclusive") + return vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "--restore-to-pos and --restore-to-timestamp are mutually exclusive") } if request.RestoreToPos != "" { pos, err := replication.DecodePosition(request.RestoreToPos) if err != nil { - return vterrors.Wrapf(err, "restore failed: unable to decode --restore_to_pos: %s", request.RestoreToPos) + return vterrors.Wrapf(err, "restore failed: unable to decode --restore-to-pos: %s", request.RestoreToPos) } params.RestoreToPos = pos } diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index d96e9116830..12284cb9beb 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -770,7 +770,7 @@ func (tm *TabletManager) handleRestore(ctx context.Context) (bool, error) { return false, fmt.Errorf("you cannot enable --restore_from_backup without a my.cnf file") } if restoreToTimestampStr != "" && restoreToPos != "" { - return false, fmt.Errorf("--restore_to_timestamp and --restore_to_pos are mutually exclusive") + return false, fmt.Errorf("--restore-to-timestamp and --restore-to-pos are mutually exclusive") } // Restore in the background @@ -795,7 +795,7 @@ func (tm *TabletManager) handleRestore(ctx context.Context) (bool, error) { var err error restoreToTimestamp, err = mysqlctl.ParseRFC3339(restoreToTimestampStr) if err != nil { - log.Exitf(fmt.Sprintf("RestoreFromBackup failed: unable to parse the --restore_to_timestamp value provided of '%s'. Error: %v", restoreToTimestampStr, err)) + log.Exitf(fmt.Sprintf("RestoreFromBackup failed: unable to parse the --restore-to-timestamp value provided of '%s'. Error: %v", restoreToTimestampStr, err)) } } // restoreFromBackup will just be a regular action From c7d1882721e7fb8ddc6103c87ceeef32f7fc4b33 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 27 Sep 2023 15:25:12 +0300 Subject: [PATCH 11/14] fix flags endtoend Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/flags/endtoend/vtcombo.txt | 4 ++-- go/flags/endtoend/vttablet.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index b5a0f0b03e9..0357932f070 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -300,11 +300,11 @@ Flags: --relay_log_max_size int Maximum buffer size (in bytes) for VReplication target buffering. If single rows are larger than this, a single row is buffered at a time. (default 250000) --remote_operation_timeout duration time to wait for a remote operation (default 15s) --replication_connect_retry duration how long to wait in between replica reconnect attempts. Only precise to the second. (default 10s) + --restore-to-pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups + --restore-to-timestamp string (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00' --restore_concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) --restore_from_backup (init restore parameter) will check BackupStorage for a recent backup at startup and start there --restore_from_backup_ts string (init restore parameter) if set, restore the latest backup taken at or before this timestamp. Example: '2021-04-29.133050' - --restore-to-pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups - --restore-to-timestamp string (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00' --retain_online_ddl_tables duration How long should vttablet keep an old migrated table before purging it (default 24h0m0s) --sanitize_log_messages Remove potentially sensitive information in tablet INFO, WARNING, and ERROR log messages such as query parameters. --schema-change-reload-timeout duration query server schema change reload timeout, this is how long to wait for the signaled schema reload operation to complete before giving up (default 30s) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index a90f93941c6..7d3a417d0e8 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -294,11 +294,11 @@ Flags: --relay_log_max_size int Maximum buffer size (in bytes) for VReplication target buffering. If single rows are larger than this, a single row is buffered at a time. (default 250000) --remote_operation_timeout duration time to wait for a remote operation (default 15s) --replication_connect_retry duration how long to wait in between replica reconnect attempts. Only precise to the second. (default 10s) + --restore-to-pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups + --restore-to-timestamp string (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00' --restore_concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) --restore_from_backup (init restore parameter) will check BackupStorage for a recent backup at startup and start there --restore_from_backup_ts string (init restore parameter) if set, restore the latest backup taken at or before this timestamp. Example: '2021-04-29.133050' - --restore-to-pos string (init incremental restore parameter) if set, run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups - --restore-to-timestamp string (init incremental restore parameter) if set, run a point in time recovery that restores up to the given timestamp, if possible. Given timestamp in RFC3339 format. Example: '2006-01-02T15:04:05Z07:00' --retain_online_ddl_tables duration How long should vttablet keep an old migrated table before purging it (default 24h0m0s) --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). --s3_backup_aws_region string AWS region to use. (default "us-east-1") From 4e32378b5ea490303527de8593d4a2a89013325a Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 28 Sep 2023 08:28:46 +0300 Subject: [PATCH 12/14] empty commit to kick ci Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> From b7cfe4d63ebb541a32496da4ae9f23e81e569b7b Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 28 Sep 2023 08:30:49 +0300 Subject: [PATCH 13/14] vtctl still uses underscores Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/backup/vtctlbackup/backup_utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index e8adbd1af18..fa390673f72 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -1303,7 +1303,7 @@ func TestReplicaRestoreToPos(t *testing.T, replicaIndex int, restoreToPos replic require.False(t, restoreToPos.IsZero()) restoreToPosArg := replication.EncodePosition(restoreToPos) - output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--", "--restore-to-pos", restoreToPosArg, replica.Alias) + output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--", "--restore_to_pos", restoreToPosArg, replica.Alias) if expectError != "" { require.Errorf(t, err, "expected: %v", expectError) require.Contains(t, output, expectError) @@ -1317,7 +1317,7 @@ func TestReplicaRestoreToPos(t *testing.T, replicaIndex int, restoreToPos replic func TestReplicaRestoreToTimestamp(t *testing.T, restoreToTimestamp time.Time, expectError string) { require.False(t, restoreToTimestamp.IsZero()) restoreToTimestampArg := mysqlctl.FormatRFC3339(restoreToTimestamp) - output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--restore-to-timestamp", restoreToTimestampArg, replica1.Alias) + output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--restore_to_timestamp", restoreToTimestampArg, replica1.Alias) if expectError != "" { require.Errorf(t, err, "expected: %v", expectError) require.Contains(t, output, expectError) From 6b667ee4072564777b9f9919966f90b96cfcac51 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 28 Sep 2023 09:26:34 +0300 Subject: [PATCH 14/14] vtctldclient uses dashes Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/backup/vtctlbackup/backup_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index fa390673f72..1ca56db68c2 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -1317,7 +1317,7 @@ func TestReplicaRestoreToPos(t *testing.T, replicaIndex int, restoreToPos replic func TestReplicaRestoreToTimestamp(t *testing.T, restoreToTimestamp time.Time, expectError string) { require.False(t, restoreToTimestamp.IsZero()) restoreToTimestampArg := mysqlctl.FormatRFC3339(restoreToTimestamp) - output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--restore_to_timestamp", restoreToTimestampArg, replica1.Alias) + output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--restore-to-timestamp", restoreToTimestampArg, replica1.Alias) if expectError != "" { require.Errorf(t, err, "expected: %v", expectError) require.Contains(t, output, expectError)