Point in time recovery (part 1)

go/cmd/vtbackup/vtbackup.go

@@ -88,9 +88,6 @@ import (
 )
 
 const (
-	backupTimestampFormat = "2006-01-02.150405"
-	manifestFileName      = "MANIFEST"
-
 	// operationTimeout is the timeout for individual operations like fetching
 	// the master position. This does not impose an overall timeout on
 	// long-running processes like taking the backup. It only applies to
@@ -154,7 +151,6 @@ func main() {
 	}()
 
 	// Open connection backup storage.
-	backupDir := fmt.Sprintf("%v/%v", *initKeyspace, *initShard)
 	backupStorage, err := backupstorage.GetBackupStorage()
 	if err != nil {
 		log.Errorf("Can't get backup storage: %v", err)
@@ -168,13 +164,14 @@ func main() {
 	// Try to take a backup, if it's been long enough since the last one.
 	// Skip pruning if backup wasn't fully successful. We don't want to be
 	// deleting things if the backup process is not healthy.
+	backupDir := mysqlctl.GetBackupDir(*initKeyspace, *initShard)
 	doBackup, err := shouldBackup(ctx, topoServer, backupStorage, backupDir)
 	if err != nil {
 		log.Errorf("Can't take backup: %v", err)
 		exit.Return(1)
 	}
 	if doBackup {
-		if err := takeBackup(ctx, topoServer, backupStorage, backupDir); err != nil {
+		if err := takeBackup(ctx, topoServer, backupStorage); err != nil {
 			log.Errorf("Failed to take backup: %v", err)
 			exit.Return(1)
 		}
@@ -187,7 +184,7 @@ func main() {
 	}
 }
 
-func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage backupstorage.BackupStorage, backupDir string) error {
+func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage backupstorage.BackupStorage) error {
 	// This is an imaginary tablet alias. The value doesn't matter for anything,
 	// except that we generate a random UID to ensure the target backup
 	// directory is unique if multiple vtbackup instances are launched for the
@@ -249,6 +246,7 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		TopoServer:   topoServer,
 		Keyspace:     *initKeyspace,
 		Shard:        *initShard,
+		TabletAlias:  topoproto.TabletAliasString(tabletAlias),
 	}
 	// In initial_backup mode, just take a backup of this empty database.
 	if *initialBackup {
@@ -265,15 +263,16 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		if err := mysqld.ExecuteSuperQueryList(ctx, cmds); err != nil {
 			return fmt.Errorf("can't initialize database: %v", err)
 		}
+		backupParams.BackupTime = time.Now()
 		// Now we're ready to take the backup.
-		name := backupName(time.Now(), tabletAlias)
-		if err := mysqlctl.Backup(ctx, backupDir, name, backupParams); err != nil {
+		if err := mysqlctl.Backup(ctx, backupParams); err != nil {
 			return fmt.Errorf("backup failed: %v", err)
 		}
 		log.Info("Initial backup successful.")
 		return nil
 	}
 
+	backupDir := mysqlctl.GetBackupDir(*initKeyspace, *initShard)
 	log.Infof("Restoring latest backup from directory %v", backupDir)
 	params := mysqlctl.RestoreParams{
 		Cnf:                 mycnf,
@@ -284,12 +283,16 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		LocalMetadata:       map[string]string{},
 		DeleteBeforeRestore: true,
 		DbName:              dbName,
-		Dir:                 backupDir,
+		Keyspace:            *initKeyspace,
+		Shard:               *initShard,
 	}
-	restorePos, err := mysqlctl.Restore(ctx, params)
+	backupManifest, err := mysqlctl.Restore(ctx, params)
+	var restorePos mysql.Position
 	switch err {
 	case nil:
 		log.Infof("Successfully restored from backup at replication position %v", restorePos)
+		// if err is nil, we expect backupManifest to be non-nil
+		restorePos = backupManifest.Position
 	case mysqlctl.ErrNoBackup:
 		// There is no backup found, but we may be taking the initial backup of a shard
 		if !*allowFirstBackup {
@@ -339,7 +342,7 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 	// Remember the time when we fetched the master position, not when we caught
 	// up to it, so the timestamp on our backup is honest (assuming we make it
 	// to the goal position).
-	backupTime := time.Now()
+	backupParams.BackupTime = time.Now()
 
 	// Wait for replication to catch up.
 	waitStartTime := time.Now()
@@ -380,8 +383,7 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 	}
 
 	// Now we can take a new backup.
-	name := backupName(backupTime, tabletAlias)
-	if err := mysqlctl.Backup(ctx, backupDir, name, backupParams); err != nil {
+	if err := mysqlctl.Backup(ctx, backupParams); err != nil {
 		return fmt.Errorf("error taking backup: %v", err)
 	}
 
@@ -490,10 +492,6 @@ func retryOnError(ctx context.Context, fn func() error) error {
 	}
 }
 
-func backupName(backupTime time.Time, tabletAlias *topodatapb.TabletAlias) string {
-	return fmt.Sprintf("%v.%v", backupTime.UTC().Format(backupTimestampFormat), topoproto.TabletAliasString(tabletAlias))
-}
-
 func pruneBackups(ctx context.Context, backupStorage backupstorage.BackupStorage, backupDir string) error {
 	if *minRetentionTime == 0 {
 		log.Info("Pruning of old backups is disabled.")
@@ -542,7 +540,7 @@ func parseBackupTime(name string) (time.Time, error) {
 	if len(parts) != 3 {
 		return time.Time{}, fmt.Errorf("backup name not in expected format (date.time.tablet-alias): %v", name)
 	}
-	backupTime, err := time.Parse(backupTimestampFormat, fmt.Sprintf("%s.%s", parts[0], parts[1]))
+	backupTime, err := time.Parse(mysqlctl.BackupTimestampFormat, fmt.Sprintf("%s.%s", parts[0], parts[1]))
 	if err != nil {
 		return time.Time{}, fmt.Errorf("can't parse timestamp from backup %q: %v", name, err)
 	}

go/vt/logutil/proto3.go

@@ -20,28 +20,29 @@ import (
 	"time"
 
 	logutilpb "vitess.io/vitess/go/vt/proto/logutil"
+	vttimepb "vitess.io/vitess/go/vt/proto/vttime"
 )
 
 // This file contains a few functions to help with proto3.
 
-// ProtoToTime converts a logutilpb.Time to a time.Time.
+// ProtoToTime converts a vttimepb.Time to a time.Time.
 // proto3 will eventually support timestamps, at which point we'll retire
 // this.
 //
 // A nil pointer is like the empty timestamp.
-func ProtoToTime(ts *logutilpb.Time) time.Time {
+func ProtoToTime(ts *vttimepb.Time) time.Time {
 	if ts == nil {
 		// treat nil like the empty Timestamp
-		return time.Unix(0, 0).UTC()
+		return time.Time{}
 	}
 	return time.Unix(ts.Seconds, int64(ts.Nanoseconds)).UTC()
 }
 
-// TimeToProto converts the time.Time to a logutilpb.Time.
-func TimeToProto(t time.Time) *logutilpb.Time {
+// TimeToProto converts the time.Time to a vttimepb.Time.
+func TimeToProto(t time.Time) *vttimepb.Time {
 	seconds := t.Unix()
 	nanos := int64(t.Sub(time.Unix(seconds, 0)))
-	return &logutilpb.Time{
+	return &vttimepb.Time{
 		Seconds:     seconds,
 		Nanoseconds: int32(nanos),
 	}

go/vt/logutil/proto3_test.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
-	logutilpb "vitess.io/vitess/go/vt/proto/logutil"
+	"vitess.io/vitess/go/vt/proto/vttime"
 )
 
 const (
@@ -39,43 +39,43 @@ func utcDate(year, month, day int) time.Time {
 }
 
 var tests = []struct {
-	pt *logutilpb.Time
+	pt *vttime.Time
 	t  time.Time
 }{
 	// The timestamp representing the Unix epoch date.
-	{pt: &logutilpb.Time{Seconds: 0, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: 0, Nanoseconds: 0},
 		t: utcDate(1970, 1, 1)},
 
 	// The smallest representable timestamp with non-negative nanos.
-	{pt: &logutilpb.Time{Seconds: math.MinInt64, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: math.MinInt64, Nanoseconds: 0},
 		t: time.Unix(math.MinInt64, 0).UTC()},
 
 	// The earliest valid timestamp.
-	{pt: &logutilpb.Time{Seconds: minValidSeconds, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: minValidSeconds, Nanoseconds: 0},
 		t: utcDate(1, 1, 1)},
 
 	// The largest representable timestamp with nanos in range.
-	{pt: &logutilpb.Time{Seconds: math.MaxInt64, Nanoseconds: 1e9 - 1},
+	{pt: &vttime.Time{Seconds: math.MaxInt64, Nanoseconds: 1e9 - 1},
 		t: time.Unix(math.MaxInt64, 1e9-1).UTC()},
 
 	// The largest valid timestamp.
-	{pt: &logutilpb.Time{Seconds: maxValidSeconds - 1, Nanoseconds: 1e9 - 1},
+	{pt: &vttime.Time{Seconds: maxValidSeconds - 1, Nanoseconds: 1e9 - 1},
 		t: time.Date(9999, 12, 31, 23, 59, 59, 1e9-1, time.UTC)},
 
 	// The smallest invalid timestamp that is larger than the valid range.
-	{pt: &logutilpb.Time{Seconds: maxValidSeconds, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: maxValidSeconds, Nanoseconds: 0},
 		t: time.Unix(maxValidSeconds, 0).UTC()},
 
 	// A date before the epoch.
-	{pt: &logutilpb.Time{Seconds: -281836800, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: -281836800, Nanoseconds: 0},
 		t: utcDate(1961, 1, 26)},
 
 	// A date after the epoch.
-	{pt: &logutilpb.Time{Seconds: 1296000000, Nanoseconds: 0},
+	{pt: &vttime.Time{Seconds: 1296000000, Nanoseconds: 0},
 		t: utcDate(2011, 1, 26)},
 
 	// A date after the epoch, in the middle of the day.
-	{pt: &logutilpb.Time{Seconds: 1296012345, Nanoseconds: 940483},
+	{pt: &vttime.Time{Seconds: 1296012345, Nanoseconds: 940483},
 		t: time.Date(2011, 1, 26, 3, 25, 45, 940483, time.UTC)},
 }