Support for encrypted backups/restore (#5079)

dgraph/cmd/alpha/admin_backup.go

@@ -109,6 +109,22 @@ func processHttpBackupRequest(ctx context.Context, r *http.Request) error {
 	req.SinceTs = latestManifest.Since
 	if forceFull {
 		req.SinceTs = 0
+	} else {
+		if worker.Config.BadgerKeyFile != "" {
+			// If encryption turned on, latest backup should be encrypted.
+			if latestManifest.Type != "" && !latestManifest.Encrypted {
+				err = errors.Errorf("latest manifest indicates the last backup was not encrypted " +
+					"but this instance has encryption turned on. Try \"forceFull\" flag.")
+				return err
+			}
+		} else {
+			// If encryption turned off, latest backup should be unencrypted.
+			if latestManifest.Type != "" && latestManifest.Encrypted {
+				err = errors.Errorf("latest manifest indicates the last backup was encrypted " +
+					"but this instance has encryption turned off. Try \"forceFull\" flag.")
+				return err
+			}
+		}
 	}
 
 	// Update the membership state to get the latest mapping of groups to predicates.
@@ -160,6 +176,9 @@ func processHttpBackupRequest(ctx context.Context, r *http.Request) error {
 		m.BackupId = latestManifest.BackupId
 		m.BackupNum = latestManifest.BackupNum + 1
 	}
+	if worker.Config.BadgerKeyFile != "" {
+		m.Encrypted = true
+	}
 
 	bp := &backup.Processor{Request: &req}
 	return bp.CompleteBackup(ctx, &m)

dgraph/cmd/alpha/run.go

@@ -36,6 +36,7 @@ import (
 	"github.com/dgraph-io/badger/v2/y"
 	"github.com/dgraph-io/dgo/v2/protos/api"
 	"github.com/dgraph-io/dgraph/edgraph"
+	"github.com/dgraph-io/dgraph/ee/backup"
 	"github.com/dgraph-io/dgraph/ee/enc"
 	"github.com/dgraph-io/dgraph/posting"
 	"github.com/dgraph-io/dgraph/schema"
@@ -520,6 +521,7 @@ func run() {
 		AuthToken:      Alpha.Conf.GetString("auth_token"),
 		AllottedMemory: Alpha.Conf.GetFloat64("lru_mb"),
 	}
+	backup.BadgerKeyFile = opts.BadgerKeyFile
 
 	// OSS, non-nil key file --> crash
 	if !enc.EeBuild && opts.BadgerKeyFile != "" {

ee/backup/backup.go

@@ -28,6 +28,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 
+	"github.com/dgraph-io/dgraph/ee/enc"
 	"github.com/dgraph-io/dgraph/posting"
 	"github.com/dgraph-io/dgraph/protos/pb"
 	"github.com/dgraph-io/dgraph/x"
@@ -77,8 +78,13 @@ type Manifest struct {
 	// Path is the path to the manifest file. This field is only used during
 	// processing and is not written to disk.
 	Path string `json:"-"`
+	// Encrypted indicates whether this backup was encrypted or not.
+	Encrypted bool `json:"encrypted"`
 }
 
+// BadgerKeyFile - This is copy of worker.Config.BadgerKeyFile. Need to copy because it results in import cycle. 
+var BadgerKeyFile string
+
 func (m *Manifest) getPredsInGroup(gid uint32) predicateSet {
 	preds, ok := m.Groups[gid]
 	if !ok {
@@ -125,7 +131,13 @@ func (pr *Processor) WriteBackup(ctx context.Context) (*pb.Status, error) {
 	}
 
 	var maxVersion uint64
-	gzWriter := gzip.NewWriter(handler)
+
+	newhandler, err := enc.GetWriter(BadgerKeyFile, handler)
+	if err != nil {
+		return &emptyRes, err
+	}
+	gzWriter := gzip.NewWriter(newhandler)
+
 	stream := pr.DB.NewStreamAt(pr.Request.ReadTs)
 	stream.LogPrefix = "Dgraph.Backup"
 	stream.KeyToList = pr.toBackupList
@@ -168,6 +180,7 @@ func (pr *Processor) WriteBackup(ctx context.Context) (*pb.Status, error) {
 		glog.Errorf("While closing gzipped writer: %v", err)
 		return &emptyRes, err
 	}
+
 	if err = handler.Close(); err != nil {
 		glog.Errorf("While closing handler: %v", err)
 		return &emptyRes, err
@@ -209,7 +222,8 @@ func (pr *Processor) CompleteBackup(ctx context.Context, manifest *Manifest) err
 
 // GoString implements the GoStringer interface for Manifest.
 func (m *Manifest) GoString() string {
-	return fmt.Sprintf(`Manifest{Since: %d, Groups: %v}`, m.Since, m.Groups)
+	return fmt.Sprintf(`Manifest{Since: %d, Groups: %v, Encrypted: %v}`,
+		m.Since, m.Groups, m.Encrypted)
 }
 
 func (pr *Processor) toBackupList(key []byte, itr *badger.Iterator) (*bpb.KVList, error) {

ee/backup/restore.go

@@ -28,13 +28,14 @@ import (
 	bpb "github.com/dgraph-io/badger/v2/pb"
 	"github.com/pkg/errors"
 
+	"github.com/dgraph-io/dgraph/ee/enc"
 	"github.com/dgraph-io/dgraph/posting"
 	"github.com/dgraph-io/dgraph/protos/pb"
 	"github.com/dgraph-io/dgraph/x"
 )
 
 // RunRestore calls badger.Load and tries to load data into a new DB.
-func RunRestore(pdir, location, backupId string) LoadResult {
+func RunRestore(pdir, location, backupId, keyfile string) LoadResult {
 	// Create the pdir if it doesn't exist.
 	if err := os.MkdirAll(pdir, 0700); err != nil {
 		return LoadResult{0, 0, err}
@@ -59,15 +60,18 @@ func RunRestore(pdir, location, backupId string) LoadResult {
 			if !pathExist(dir) {
 				fmt.Println("Creating new db:", dir)
 			}
+			r, err = enc.GetReader(keyfile, r)
+			if err != nil {
+				return 0, err
+			}
 			gzReader, err := gzip.NewReader(r)
 			if err != nil {
-				return 0, nil
+				return 0, err
 			}
 			maxUid, err := loadFromBackup(db, gzReader, preds)
 			if err != nil {
 				return 0, err
 			}
-
 			return maxUid, x.WriteGroupIdFile(dir, uint32(groupId))
 		})
 }

ee/backup/run.go

@@ -32,7 +32,7 @@ var Restore x.SubCommand
 var LsBackup x.SubCommand
 
 var opt struct {
-	backupId, location, pdir, zero string
+	backupId, location, pdir, zero, keyfile string
 }
 
 func init() {
@@ -104,6 +104,8 @@ $ dgraph restore -p . -l /var/backups/dgraph -z localhost:5080
 	flag.StringVarP(&opt.zero, "zero", "z", "", "gRPC address for Dgraph zero. ex: localhost:5080")
 	flag.StringVarP(&opt.backupId, "backup_id", "", "", "The ID of the backup series to "+
 		"restore. If empty, it will restore the latest series.")
+	flag.StringVarP(&opt.keyfile, "keyfile", "k", "",
+		"Key file to decrypt the backup")
 	_ = Restore.Cmd.MarkFlagRequired("postings")
 	_ = Restore.Cmd.MarkFlagRequired("location")
 }
@@ -184,7 +186,7 @@ func runRestoreCmd() error {
 	}
 
 	start = time.Now()
-	result := RunRestore(opt.pdir, opt.location, opt.backupId)
+	result := RunRestore(opt.pdir, opt.location, opt.backupId, opt.keyfile)
 	if result.Err != nil {
 		return result.Err
 	}
@@ -225,9 +227,9 @@ func runLsbackupCmd() error {
 		return errors.Wrapf(err, "while listing manifests")
 	}
 
-	fmt.Printf("Name\tSince\tGroups\n")
+	fmt.Printf("Name\tSince\tGroups\tEncrypted\n")
 	for path, manifest := range manifests {
-		fmt.Printf("%v\t%v\t%v\n", path, manifest.Since, manifest.Groups)
+		fmt.Printf("%v\t%v\t%v\t%v\n", path, manifest.Since, manifest.Groups, manifest.Encrypted)
 	}
 
 	return nil