[BREAKING]: Change how Badger handles WAL

backup_test.go

@@ -102,6 +102,7 @@ func TestBackupRestore1(t *testing.T) {
 			if err != nil {
 				return err
 			}
+			t.Logf("Got entry: %v\n", item.Version())
 			require.Equal(t, entries[count].key, item.Key())
 			require.Equal(t, entries[count].val, val)
 			require.Equal(t, entries[count].version, item.Version())
@@ -112,7 +113,7 @@ func TestBackupRestore1(t *testing.T) {
 		return nil
 	})
 	require.NoError(t, err)
-	require.Equal(t, db.orc.nextTs(), uint64(3))
+	require.Equal(t, 3, int(db.orc.nextTs()))
 }
 
 func TestBackupRestore2(t *testing.T) {

badger/cmd/backup.go

@@ -26,7 +26,6 @@ import (
 )
 
 var backupFile string
-var truncate bool
 
 // backupCmd represents the backup command
 var backupCmd = &cobra.Command{
@@ -45,16 +44,13 @@ func init() {
 	RootCmd.AddCommand(backupCmd)
 	backupCmd.Flags().StringVarP(&backupFile, "backup-file", "f",
 		"badger.bak", "File to backup to")
-	backupCmd.Flags().BoolVarP(&truncate, "truncate", "t",
-		false, "Allow value log truncation if required.")
 	backupCmd.Flags().IntVarP(&numVersions, "num-versions", "n",
 		0, "Number of versions to keep. A value <= 0 means keep all versions.")
 }
 
 func doBackup(cmd *cobra.Command, args []string) error {
 	opt := badger.DefaultOptions(sstDir).
 		WithValueDir(vlogDir).
-		WithTruncate(truncate).
 		WithNumVersionsToKeep(math.MaxInt32)
 
 	if numVersions > 0 {

badger/cmd/bank.go

@@ -364,11 +364,10 @@ func runTest(cmd *cobra.Command, args []string) error {
 		WithNumVersionsToKeep(int(math.MaxInt32)).
 		WithValueThreshold(1). // Make all values go to value log
 		WithCompression(options.ZSTD).
-		WithKeepL0InMemory(false).
 		WithBlockCacheSize(10 << 20)
 
-	if mmap {
-		opts = opts.WithTableLoadingMode(options.MemoryMap)
+	if verbose {
+		opts = opts.WithLoggingLevel(badger.DEBUG)
 	}
 
 	if encryptionKey != "" {

badger/cmd/flatten.go

@@ -52,7 +52,6 @@ func flatten(cmd *cobra.Command, args []string) error {
 	}
 	opt := badger.DefaultOptions(sstDir).
 		WithValueDir(vlogDir).
-		WithTruncate(truncate).
 		WithNumVersionsToKeep(numVersions).
 		WithNumCompactors(0)
 	fmt.Printf("Opening badger with options = %+v\n", opt)

badger/cmd/info.go

@@ -31,7 +31,6 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
 	"github.com/dgraph-io/badger/v2/table"
 	"github.com/dgraph-io/badger/v2/y"
 	humanize "github.com/dustin/go-humanize"
@@ -93,18 +92,16 @@ to the Dgraph team.
 
 func handleInfo(cmd *cobra.Command, args []string) error {
 	if err := printInfo(sstDir, vlogDir); err != nil {
-		return errors.Wrap(err, "failed to print information in MANIFEST file")
+		return y.Wrap(err, "failed to print information in MANIFEST file")
 	}
 
 	// Open DB
 	db, err := badger.Open(badger.DefaultOptions(sstDir).
 		WithValueDir(vlogDir).
 		WithReadOnly(opt.readOnly).
-		WithTruncate(opt.truncate).
-		WithTableLoadingMode(options.MemoryMap).
 		WithEncryptionKey([]byte(opt.encryptionKey)))
 	if err != nil {
-		return errors.Wrap(err, "failed to open database")
+		return y.Wrap(err, "failed to open database")
 	}
 	defer db.Close()
 
@@ -114,7 +111,7 @@ func handleInfo(cmd *cobra.Command, args []string) error {
 
 	prefix, err := hex.DecodeString(opt.withPrefix)
 	if err != nil {
-		return errors.Wrapf(err, "failed to decode hex prefix: %s", opt.withPrefix)
+		return y.Wrapf(err, "failed to decode hex prefix: %s", opt.withPrefix)
 	}
 	if opt.showHistogram {
 		db.PrintHistogram(prefix)
@@ -128,7 +125,7 @@ func handleInfo(cmd *cobra.Command, args []string) error {
 
 	if len(opt.keyLookup) > 0 {
 		if err := lookup(db); err != nil {
-			return errors.Wrapf(err, "failed to perform lookup for the key: %x", opt.keyLookup)
+			return y.Wrapf(err, "failed to perform lookup for the key: %x", opt.keyLookup)
 		}
 	}
 	return nil
@@ -153,7 +150,7 @@ func showKeys(db *badger.DB, prefix []byte) error {
 	for it.Rewind(); it.Valid(); it.Next() {
 		item := it.Item()
 		if err := printKey(item, false); err != nil {
-			return errors.Wrapf(err, "failed to print information about key: %x(%d)",
+			return y.Wrapf(err, "failed to print information about key: %x(%d)",
 				item.Key(), item.Version())
 		}
 		totalKeys++
@@ -170,7 +167,7 @@ func lookup(db *badger.DB) error {
 
 	key, err := hex.DecodeString(opt.keyLookup)
 	if err != nil {
-		return errors.Wrapf(err, "failed to decode key: %q", opt.keyLookup)
+		return y.Wrapf(err, "failed to decode key: %q", opt.keyLookup)
 	}
 
 	iopts := badger.DefaultIteratorOptions
@@ -186,7 +183,7 @@ func lookup(db *badger.DB) error {
 	fmt.Println()
 	item := itr.Item()
 	if err := printKey(item, true); err != nil {
-		return errors.Wrapf(err, "failed to print information about key: %x(%d)",
+		return y.Wrapf(err, "failed to print information about key: %x(%d)",
 			item.Key(), item.Version())
 	}
 
@@ -201,7 +198,7 @@ func lookup(db *badger.DB) error {
 			break
 		}
 		if err := printKey(item, true); err != nil {
-			return errors.Wrapf(err, "failed to print information about key: %x(%d)",
+			return y.Wrapf(err, "failed to print information about key: %x(%d)",
 				item.Key(), item.Version())
 		}
 	}
@@ -223,7 +220,7 @@ func printKey(item *badger.Item, showValue bool) error {
 	if showValue {
 		val, err := item.ValueCopy(nil)
 		if err != nil {
-			return errors.Wrapf(err,
+			return y.Wrapf(err,
 				"failed to copy value of the key: %x(%d)", item.Key(), item.Version())
 		}
 		fmt.Fprintf(&buf, "\n\tvalue: %v", val)

badger/cmd/read_bench.go

@@ -21,15 +21,13 @@ import (
 	"fmt"
 	"math"
 	"math/rand"
-	"strings"
 	"sync/atomic"
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/spf13/cobra"
 
 	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
 	"github.com/dgraph-io/badger/v2/pb"
 	"github.com/dgraph-io/badger/v2/y"
 	"github.com/dgraph-io/ristretto/z"
@@ -107,12 +105,9 @@ func readBench(cmd *cobra.Command, args []string) error {
 		return y.Wrapf(err, "unable to parse duration")
 	}
 	y.AssertTrue(numGoroutines > 0)
-	mode := getLoadingMode(loadingMode)
 	opt := badger.DefaultOptions(sstDir).
 		WithValueDir(vlogDir).
 		WithReadOnly(readOnly).
-		WithTableLoadingMode(mode).
-		WithValueLogLoadingMode(mode).
 		WithBlockCacheSize(blockCacheSize << 20).
 		WithIndexCacheSize(indexCacheSize << 20)
 	fmt.Printf("Opening badger with options = %+v\n", opt)
@@ -239,18 +234,3 @@ func getSampleKeys(db *badger.DB) ([][]byte, error) {
 
 	return keys, nil
 }
-
-func getLoadingMode(m string) options.FileLoadingMode {
-	m = strings.ToLower(m)
-	var mode options.FileLoadingMode
-	switch m {
-	case "fileio":
-		mode = options.FileIO
-	case "mmap":
-		mode = options.MemoryMap
-	default:
-		panic("loading mode not supported")
-	}
-
-	return mode
-}

badger/cmd/rotate_test.go

@@ -31,10 +31,12 @@ func TestRotate(t *testing.T) {
 	dir, err := ioutil.TempDir("", "badger-test")
 	require.NoError(t, err)
 	defer os.RemoveAll(dir)
+
 	// Creating sample key.
 	key := make([]byte, 32)
 	_, err = rand.Read(key)
 	require.NoError(t, err)
+
 	fp, err := ioutil.TempFile("", "*.key")
 	require.NoError(t, err)
 	_, err = fp.Write(key)
@@ -44,6 +46,8 @@ func TestRotate(t *testing.T) {
 	// Opening DB with the encryption key.
 	opts := badger.DefaultOptions(dir)
 	opts.EncryptionKey = key
+	opts.BlockCacheSize = 1 << 20
+
 	db, err := badger.Open(opts)
 	require.NoError(t, err)
 	// Closing the db.
@@ -126,6 +130,7 @@ func TestRotatePlainTextToEncrypted(t *testing.T) {
 	require.Nil(t, doRotate(nil, []string{}))
 
 	// Try opening DB without the key.
+	opts.BlockCacheSize = 1 << 20
 	_, err = badger.Open(opts)
 	require.EqualError(t, err, badger.ErrEncryptionKeyMismatch.Error())
 

badger/cmd/stream.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/dgraph-io/badger/v2"
 	"github.com/dgraph-io/badger/v2/options"
+	"github.com/dgraph-io/badger/v2/y"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
@@ -44,7 +45,6 @@ func init() {
 	RootCmd.AddCommand(streamCmd)
 	streamCmd.Flags().StringVarP(&outDir, "out", "o", "",
 		"Path to output DB. The directory should be empty.")
-	streamCmd.Flags().BoolVarP(&truncate, "truncate", "", false, "Option to truncate the DBs")
 	streamCmd.Flags().BoolVarP(&readOnly, "read_only", "", true,
 		"Option to open input DB in read-only mode")
 	streamCmd.Flags().IntVarP(&numVersions, "num_versions", "", 0,
@@ -76,7 +76,6 @@ func stream(cmd *cobra.Command, args []string) error {
 	}
 	inOpt := badger.DefaultOptions(sstDir).
 		WithReadOnly(readOnly).
-		WithTruncate(truncate).
 		WithValueThreshold(1 << 10 /* 1KB */).
 		WithNumVersionsToKeep(numVersions)
 
@@ -90,7 +89,7 @@ func stream(cmd *cobra.Command, args []string) error {
 
 	inDB, err := badger.OpenManaged(inOpt)
 	if err != nil {
-		return errors.Wrapf(err, "cannot open DB at %s", sstDir)
+		return y.Wrapf(err, "cannot open DB at %s", sstDir)
 	}
 	defer inDB.Close()
 	return inDB.StreamDB(outOpt)