add vlogmaxsize config option

cli/start.go

@@ -235,6 +235,15 @@ func init() {
 		log.FeedbackFatalE(context.Background(), "Could not bind datastore.store", err)
 	}
 
+	startCmd.Flags().Var(
+		&cfg.Datastore.Badger.VLogMaxSize, "vlogmaxsize",
+		"Specify the datastore maximum value log file size (in bytes)",
+	)
+	err = viper.BindPFlag("datastore.badger.vlogmaxsize", startCmd.Flags().Lookup("vlogmaxsize"))
+	if err != nil {
+		log.FeedbackFatalE(context.Background(), "Could not bind datastore..badger.vlogmaxsize", err)
+	}
+
 	startCmd.Flags().String(
 		"p2paddr", cfg.Net.P2PAddress,
 		"Listener address for the p2p network (formatted as a libp2p MultiAddr)",

config/config.go

@@ -50,10 +50,13 @@ import (
 	"fmt"
 	"net"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
+	"unicode"
 
+	"github.com/mitchellh/mapstructure"
 	ma "github.com/multiformats/go-multiaddr"
 	badgerds "github.com/sourcenetwork/defradb/datastore/badger/v3"
 	"github.com/sourcenetwork/defradb/logging"
@@ -94,9 +97,10 @@ func (cfg *Config) Load(rootDirPath string) error {
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.AutomaticEnv()
 
-	if err := viper.Unmarshal(cfg); err != nil {
+	if err := viper.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())); err != nil {
 		return err
 	}
+	cfg.setBadgerVLogMaxSize()
 	cfg.handleParams(rootDirPath)
 	err := cfg.validate()
 	if err != nil {
@@ -124,13 +128,14 @@ func (cfg *Config) LoadWithoutRootDir() error {
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.AutomaticEnv()
 
-	if err := viper.Unmarshal(cfg); err != nil {
+	if err := viper.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())); err != nil {
 		return err
 	}
 	rootDir, err := DefaultRootDir()
 	if err != nil {
 		log.FatalE(context.Background(), "Could not get home directory", err)
 	}
+	cfg.setBadgerVLogMaxSize()
 	cfg.handleParams(rootDir)
 	err = cfg.validate()
 	if err != nil {
@@ -172,6 +177,10 @@ func (cfg *Config) handleParams(rootDir string) {
 	}
 }
 
+func (cfg *Config) setBadgerVLogMaxSize() {
+	cfg.Datastore.Badger.ValueLogFileSize = int64(cfg.Datastore.Badger.VLogMaxSize)
+}
+
 // DatastoreConfig configures datastores.
 type DatastoreConfig struct {
 	Store  string
@@ -181,20 +190,97 @@ type DatastoreConfig struct {
 
 // BadgerConfig configures Badger's on-disk / filesystem mode.
 type BadgerConfig struct {
-	Path string
+	Path        string
+	VLogMaxSize ByteSize
 	*badgerds.Options
 }
 
+type ByteSize uint64
+
+const (
+	B  ByteSize = 1
+	KB          = B << 10
+	MB          = KB << 10
+	GB          = MB << 10
+	TB          = GB << 10
+	PB          = TB << 10
+)
+
+// UnmarshallText calls Set on ByteSize with the given text
+func (bs *ByteSize) UnmarshalText(text []byte) error {
+	return bs.Set(string(text))
+}
+
+// Set parses a string into ByteSize
+func (bs *ByteSize) Set(s string) error {
+	digitString := ""
+	unit := ""
+	for _, char := range s {
+		if unicode.IsDigit(char) {
+			digitString += string(char)
+		} else {
+			unit += string(char)
+		}
+	}
+	digits, err := strconv.Atoi(digitString)
+	if err != nil {
+		return err
+	}
+
+	switch strings.ToUpper(strings.Trim(unit, " ")) {
+	case "B":
+		*bs = ByteSize(digits) * B
+	case "KB":
+		*bs = ByteSize(digits) * KB
+	case "MB":
+		*bs = ByteSize(digits) * MB
+	case "GB":
+		*bs = ByteSize(digits) * GB
+	case "TB":
+		*bs = ByteSize(digits) * TB
+	case "PB":
+		*bs = ByteSize(digits) * PB
+	default:
+		*bs = ByteSize(digits)
+	}
+
+	return nil
+}
+
+// String returns the string formatted output of ByteSize
+func (bs *ByteSize) String() string {
+	const unit = 1000
+	bsInt := int64(*bs)
+	if bsInt < unit {
+		return fmt.Sprintf("%d", bsInt)
+	}
+	div, exp := int64(unit), 0
+	for n := bsInt / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%d%cB", bsInt/div, "KMGTP"[exp])
+}
+
+// Type returns the type as a string.
+func (bs *ByteSize) Type() string {
+	return "ByteSize"
+}
+
 // MemoryConfig configures of Badger's memory mode.
 type MemoryConfig struct {
 	Size uint64
 }
 
 func defaultDatastoreConfig() *DatastoreConfig {
+	// create a copy of the default badger options
+	opts := badgerds.DefaultOptions
 	return &DatastoreConfig{
 		Store: "badger",
 		Badger: BadgerConfig{
-			Path: "data",
+			Path:        "data",
+			VLogMaxSize: 1 * GB,
+			Options:     &opts,
 		},
 	}
 }

config/config_test.go

@@ -338,3 +338,87 @@ func TestNodeConfig(t *testing.T) {
 	assert.Equal(t, expectedOptions.EnablePubSub, options.EnablePubSub)
 	assert.Equal(t, expectedOptions.EnableRelay, options.EnableRelay)
 }
+
+func TestUnmarshallByteSize(t *testing.T) {
+	var bs ByteSize
+
+	b := []byte("10")
+	err := bs.UnmarshalText(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*B, bs)
+
+	b = []byte("10B")
+	err = bs.UnmarshalText(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*B, bs)
+
+	b = []byte("10 B")
+	err = bs.UnmarshalText(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*B, bs)
+
+	kb := []byte("10KB")
+	err = bs.UnmarshalText(kb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*KB, bs)
+
+	kb = []byte("10 kb")
+	err = bs.UnmarshalText(kb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*KB, bs)
+
+	mb := []byte("10MB")
+	err = bs.UnmarshalText(mb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*MB, bs)
+
+	gb := []byte("10GB")
+	err = bs.UnmarshalText(gb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*GB, bs)
+
+	tb := []byte("10TB")
+	err = bs.UnmarshalText(tb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*TB, bs)
+
+	pb := []byte("10PB")
+	err = bs.UnmarshalText(pb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*PB, bs)
+
+	eb := []byte("१")
+	err = bs.UnmarshalText(eb)
+	assert.Error(t, err)
+}
+
+func TestByteSizeType(t *testing.T) {
+	var bs ByteSize
+	assert.Equal(t, "ByteSize", bs.Type())
+}
+
+func TestByteSizeToString(t *testing.T) {
+	b := 999 * B
+	assert.Equal(t, "999", b.String())
+
+	mb := 10 * MB
+	assert.Equal(t, "10MB", mb.String())
+}

config/configfile.go

@@ -54,6 +54,9 @@ datastore:
   store: {{ .Datastore.Store }}
   badger:
     path: {{ .Datastore.Badger.Path }}
+    # Maximum file size of the value log file. The actual file size will be 2*vlogmaxsize.
+    # Human friendly units can be used (ex: 500MB).
+    vlogmaxsize: {{ .Datastore.Badger.VLogMaxSize }}
   # memory:
   #    size: {{ .Datastore.Memory.Size }}
 

config/configfile_test.go

@@ -12,6 +12,7 @@ package config
 
 import (
 	"bytes"
+	"fmt"
 	"os"
 	"testing"
 	"text/template"
@@ -102,3 +103,35 @@ func TestReadConfigFileForLogger(t *testing.T) {
 	assert.Equal(t, cfg.Log.Output, cfgFromFile.Log.Output)
 	assert.Equal(t, cfg.Log.Stacktrace, cfgFromFile.Log.Stacktrace)
 }
+
+func TestReadConfigFileForDatastore(t *testing.T) {
+	dir := t.TempDir()
+
+	cfg := DefaultConfig()
+	cfg.Datastore.Store = "badger"
+	cfg.Datastore.Badger.Path = "dataPath"
+	cfg.Datastore.Badger.VLogMaxSize = 512 * MB
+
+	err := cfg.WriteConfigFileToRootDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	path := dir + "/" + DefaultDefraDBConfigFileName
+
+	_, err = os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cfgFromFile := DefaultConfig()
+
+	err = cfgFromFile.Load(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(cfg.Datastore.Badger.VLogMaxSize, cfgFromFile.Datastore.Badger.VLogMaxSize)
+	assert.Equal(t, cfg.Datastore.Store, cfgFromFile.Datastore.Store)
+	assert.Equal(t, dir+"/"+cfg.Datastore.Badger.Path, cfgFromFile.Datastore.Badger.Path)
+	assert.NotEqual(t, cfg.Datastore.Badger.VLogMaxSize, cfgFromFile.Datastore.Badger.VLogMaxSize)
+}