Point-in-time-recovery

go/vt/logutil/proto3.go

@@ -20,28 +20,29 @@ import (
 	"time"
 
 	logutilpb "vitess.io/vitess/go/vt/proto/logutil"
+	vttimepb "vitess.io/vitess/go/vt/proto/vttime"
 )
 
 // This file contains a few functions to help with proto3.
 
-// ProtoToTime converts a logutilpb.Time to a time.Time.
+// ProtoToTime converts a vttimepb.Time to a time.Time.
 // proto3 will eventually support timestamps, at which point we'll retire
 // this.
 //
 // A nil pointer is like the empty timestamp.
-func ProtoToTime(ts *logutilpb.Time) time.Time {
+func ProtoToTime(ts *vttimepb.Time) time.Time {
 	if ts == nil {
 		// treat nil like the empty Timestamp
 		return time.Unix(0, 0).UTC()
 	}
 	return time.Unix(ts.Seconds, int64(ts.Nanoseconds)).UTC()
 }
 
-// TimeToProto converts the time.Time to a logutilpb.Time.
-func TimeToProto(t time.Time) *logutilpb.Time {
+// TimeToProto converts the time.Time to a vttimepb.Time.
+func TimeToProto(t time.Time) *vttimepb.Time {
 	seconds := t.Unix()
 	nanos := int64(t.Sub(time.Unix(seconds, 0)))
-	return &logutilpb.Time{
+	return &vttimepb.Time{
 		Seconds:     seconds,
 		Nanoseconds: int32(nanos),
 	}

go/vt/logutil/proto3_test.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
-	logutilpb "vitess.io/vitess/go/vt/proto/logutil"
+	"vitess.io/vitess/go/vt/proto/vttime"
 )
 
 const (
@@ -39,43 +39,43 @@ func utcDate(year, month, day int) time.Time {
 }
 
 var tests = []struct {
-	pt *logutilpb.Time
+	pt *vttime.Time
 	t  time.Time
 }{
 	// The timestamp representing the Unix epoch date.
-	{pt: &logutilpb.Time{Seconds: 0, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: 0, Nanoseconds: 0},
 		t: utcDate(1970, 1, 1)},
 
 	// The smallest representable timestamp with non-negative nanos.
-	{pt: &logutilpb.Time{Seconds: math.MinInt64, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: math.MinInt64, Nanoseconds: 0},
 		t: time.Unix(math.MinInt64, 0).UTC()},
 
 	// The earliest valid timestamp.
-	{pt: &logutilpb.Time{Seconds: minValidSeconds, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: minValidSeconds, Nanoseconds: 0},
 		t: utcDate(1, 1, 1)},
 
 	// The largest representable timestamp with nanos in range.
-	{pt: &logutilpb.Time{Seconds: math.MaxInt64, Nanoseconds: 1e9 - 1},
+	{pt: &vttime.Time{Seconds: math.MaxInt64, Nanoseconds: 1e9 - 1},
 		t: time.Unix(math.MaxInt64, 1e9-1).UTC()},
 
 	// The largest valid timestamp.
-	{pt: &logutilpb.Time{Seconds: maxValidSeconds - 1, Nanoseconds: 1e9 - 1},
+	{pt: &vttime.Time{Seconds: maxValidSeconds - 1, Nanoseconds: 1e9 - 1},
 		t: time.Date(9999, 12, 31, 23, 59, 59, 1e9-1, time.UTC)},
 
 	// The smallest invalid timestamp that is larger than the valid range.
-	{pt: &logutilpb.Time{Seconds: maxValidSeconds, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: maxValidSeconds, Nanoseconds: 0},
 		t: time.Unix(maxValidSeconds, 0).UTC()},
 
 	// A date before the epoch.
-	{pt: &logutilpb.Time{Seconds: -281836800, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: -281836800, Nanoseconds: 0},
 		t: utcDate(1961, 1, 26)},
 
 	// A date after the epoch.
-	{pt: &logutilpb.Time{Seconds: 1296000000, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: 1296000000, Nanoseconds: 0},
 		t: utcDate(2011, 1, 26)},
 
 	// A date after the epoch, in the middle of the day.
-	{pt: &logutilpb.Time{Seconds: 1296012345, Nanoseconds: 940483},
+	{pt: &vttime.Time{Seconds: 1296012345, Nanoseconds: 940483},
 		t: time.Date(2011, 1, 26, 3, 25, 45, 940483, time.UTC)},
 }
 

go/vt/mysqlctl/backup.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"golang.org/x/net/context"
 
@@ -83,7 +84,7 @@ var (
 // - uses the BackupStorage service to store a new backup
 // - shuts down Mysqld during the backup
 // - remember if we were replicating, restore the exact same state
-func Backup(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, dir, name string, backupConcurrency int, hookExtraEnv map[string]string) error {
+func Backup(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, dir, name string, backupConcurrency int, hookExtraEnv map[string]string, backupTime time.Time) error {
 	// Start the backup with the BackupStorage.
 	bs, err := backupstorage.GetBackupStorage()
 	if err != nil {
@@ -101,7 +102,7 @@ func Backup(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.
 	}
 
 	// Take the backup, and either AbortBackup or EndBackup.
-	usable, err := be.ExecuteBackup(ctx, cnf, mysqld, logger, bh, backupConcurrency, hookExtraEnv)
+	usable, err := be.ExecuteBackup(ctx, cnf, mysqld, logger, bh, backupConcurrency, hookExtraEnv, backupTime)
 	var finishErr error
 	if usable {
 		finishErr = bh.EndBackup(ctx)
@@ -218,41 +219,44 @@ func Restore(
 	localMetadata map[string]string,
 	logger logutil.Logger,
 	deleteBeforeRestore bool,
-	dbName string) (mysql.Position, error) {
+	dbName string,
+	snapshotTime time.Time) (mysql.Position, string, error) {
 
-	rval := mysql.Position{}
+	var zeroPosition, rval mysql.Position
+	zeroPosition = mysql.Position{}
+	var t, backupTime string
 
 	// Wait for mysqld to be ready, in case it was launched in parallel with us.
 	if err := mysqld.Wait(ctx, cnf); err != nil {
-		return mysql.Position{}, err
+		return zeroPosition, t, err
 	}
 
 	if !deleteBeforeRestore {
 		logger.Infof("Restore: checking no existing data is present")
 		ok, err := checkNoDB(ctx, mysqld, dbName)
 		if err != nil {
-			return mysql.Position{}, err
+			return zeroPosition, t, err
 		}
 		if !ok {
 			logger.Infof("Auto-restore is enabled, but mysqld already contains data. Assuming vttablet was just restarted.")
 			if err = PopulateMetadataTables(mysqld, localMetadata, dbName); err == nil {
 				err = ErrExistingDB
 			}
-			return mysql.Position{}, err
+			return zeroPosition, t, err
 		}
 	}
 
 	// find the right backup handle: most recent one, with a MANIFEST
 	logger.Infof("Restore: looking for a suitable backup to restore")
 	bs, err := backupstorage.GetBackupStorage()
 	if err != nil {
-		return mysql.Position{}, err
+		return zeroPosition, t, err
 	}
 	defer bs.Close()
 
 	bhs, err := bs.ListBackups(ctx, dir)
 	if err != nil {
-		return mysql.Position{}, vterrors.Wrap(err, "ListBackups failed")
+		return zeroPosition, t, vterrors.Wrap(err, "ListBackups failed")
 	}
 
 	if len(bhs) == 0 {
@@ -267,15 +271,15 @@ func Restore(
 		if err2 := PopulateMetadataTables(mysqld, localMetadata, dbName); err2 == nil {
 			err = ErrNoBackup
 		}
-		return mysql.Position{}, err
+		return zeroPosition, t, err
 	}
 
 	be, err := GetBackupEngine()
 	if err != nil {
-		return mysql.Position{}, vterrors.Wrap(err, "Failed to find backup engine")
+		return zeroPosition, t, vterrors.Wrap(err, "Failed to find backup engine")
 	}
-	if rval, err = be.ExecuteRestore(ctx, cnf, mysqld, logger, dir, bhs, restoreConcurrency, hookExtraEnv); err != nil {
-		return rval, err
+	if rval, backupTime, err = be.ExecuteRestore(ctx, cnf, mysqld, logger, dir, bhs, restoreConcurrency, hookExtraEnv, snapshotTime); err != nil {
+		return rval, backupTime, err
 	}
 
 	// mysqld needs to be running in order for mysql_upgrade to work.
@@ -289,33 +293,33 @@ func Restore(
 	// Note Start will use dba user for waiting, this is fine, it will be allowed.
 	err = mysqld.Start(context.Background(), cnf, "--skip-grant-tables", "--skip-networking")
 	if err != nil {
-		return mysql.Position{}, err
+		return zeroPosition, t, err
 	}
 
 	logger.Infof("Restore: running mysql_upgrade")
 	if err := mysqld.RunMysqlUpgrade(); err != nil {
-		return mysql.Position{}, vterrors.Wrap(err, "mysql_upgrade failed")
+		return zeroPosition, t, vterrors.Wrap(err, "mysql_upgrade failed")
 	}
 
 	// Populate local_metadata before starting without --skip-networking,
 	// so it's there before we start announcing ourselves.
 	logger.Infof("Restore: populating local_metadata")
 	err = PopulateMetadataTables(mysqld, localMetadata, dbName)
 	if err != nil {
-		return mysql.Position{}, err
+		return zeroPosition, t, err
 	}
 
 	// The MySQL manual recommends restarting mysqld after running mysql_upgrade,
 	// so that any changes made to system tables take effect.
 	logger.Infof("Restore: restarting mysqld after mysql_upgrade")
 	err = mysqld.Shutdown(context.Background(), cnf, true)
 	if err != nil {
-		return mysql.Position{}, err
+		return zeroPosition, t, err
 	}
 	err = mysqld.Start(context.Background(), cnf)
 	if err != nil {
-		return mysql.Position{}, err
+		return zeroPosition, t, err
 	}
 
-	return rval, nil
+	return rval, backupTime, nil
 }

go/vt/mysqlctl/backupengine.go

@@ -19,6 +19,7 @@ package mysqlctl
 import (
 	"context"
 	"flag"
+	"time"
 
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/vt/logutil"
@@ -34,8 +35,8 @@ var (
 
 // BackupEngine is the interface to the backup engine
 type BackupEngine interface {
-	ExecuteBackup(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, backupConcurrency int, hookExtraEnv map[string]string) (bool, error)
-	ExecuteRestore(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, dir string, bhs []backupstorage.BackupHandle, restoreConcurrency int, hookExtraEnv map[string]string) (mysql.Position, error)
+	ExecuteBackup(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, backupConcurrency int, hookExtraEnv map[string]string, backupTime time.Time) (bool, error)
+	ExecuteRestore(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, dir string, bhs []backupstorage.BackupHandle, restoreConcurrency int, hookExtraEnv map[string]string, snapshotTime time.Time) (mysql.Position, string, error)
 }
 
 // BackupEngineMap contains the registered implementations for BackupEngine