Skip to content

Commit

Permalink
Export backup loader interface (#901)
Browse files Browse the repository at this point in the history
The logic in the load method will be copied to Dgraph so that it can
support converting keys and values from a different format. To
allow this, the loader interface has been exported as KVLoader.

Also add a note telling users to use the KVLoader if more complex logic
is needed to restore a backup.
  • Loading branch information
martinmr authored Jul 1, 2019
1 parent d562bfa commit 50ccc86
Showing 1 changed file with 15 additions and 10 deletions.
25 changes: 15 additions & 10 deletions backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,20 +127,23 @@ func writeTo(list *pb.KVList, w io.Writer) error {
return err
}

type loader struct {
// KVLoader is used to write KVList objects in to badger. It can be used to restore a backup.
type KVLoader struct {
db *DB
throttle *y.Throttle
entries []*Entry
}

func (db *DB) newLoader(maxPendingWrites int) *loader {
return &loader{
// NewKVLoader returns a new instance of KVLoader.
func (db *DB) NewKVLoader(maxPendingWrites int) *KVLoader {
return &KVLoader{
db: db,
throttle: y.NewThrottle(maxPendingWrites),
}
}

func (l *loader) set(kv *pb.KV) error {
// Set writes the key-value pair to the database.
func (l *KVLoader) Set(kv *pb.KV) error {
var userMeta, meta byte
if len(kv.UserMeta) > 0 {
userMeta = kv.UserMeta[0]
Expand All @@ -162,7 +165,7 @@ func (l *loader) set(kv *pb.KV) error {
return nil
}

func (l *loader) send() error {
func (l *KVLoader) send() error {
if err := l.throttle.Do(); err != nil {
return err
}
Expand All @@ -176,7 +179,8 @@ func (l *loader) send() error {
return nil
}

func (l *loader) finish() error {
// Finish is meant to be called after all the key-value pairs have been loaded.
func (l *KVLoader) Finish() error {
if len(l.entries) > 0 {
if err := l.send(); err != nil {
return err
Expand All @@ -187,15 +191,16 @@ func (l *loader) finish() error {

// Load reads a protobuf-encoded list of all entries from a reader and writes
// them to the database. This can be used to restore the database from a backup
// made by calling DB.Backup().
// made by calling DB.Backup(). If more complex logic is needed to restore a badger
// backup, the KVLoader interface should be used instead.
//
// DB.Load() should be called on a database that is not running any other
// concurrent transactions while it is running.
func (db *DB) Load(r io.Reader, maxPendingWrites int) error {
br := bufio.NewReaderSize(r, 16<<10)
unmarshalBuf := make([]byte, 1<<10)

ldr := db.newLoader(maxPendingWrites)
ldr := db.NewKVLoader(maxPendingWrites)
for {
var sz uint64
err := binary.Read(br, binary.LittleEndian, &sz)
Expand All @@ -219,7 +224,7 @@ func (db *DB) Load(r io.Reader, maxPendingWrites int) error {
}

for _, kv := range list.Kv {
if err := ldr.set(kv); err != nil {
if err := ldr.Set(kv); err != nil {
return err
}

Expand All @@ -231,7 +236,7 @@ func (db *DB) Load(r io.Reader, maxPendingWrites int) error {
}
}

if err := ldr.finish(); err != nil {
if err := ldr.Finish(); err != nil {
return err
}
db.orc.txnMark.Done(db.orc.nextTxnTs - 1)
Expand Down

0 comments on commit 50ccc86

Please sign in to comment.