feat: Add a config option to set the vlog max file size (sourcenetwor…

…k#743)

Description
This PR adds a config option that gives the user the ability to set the maximum value log file size for the badger DB datastore.

cli/start.go

@@ -235,6 +235,15 @@ func init() {
 		log.FeedbackFatalE(context.Background(), "Could not bind datastore.store", err)
 	}
 
+	startCmd.Flags().Var(
+		&cfg.Datastore.Badger.ValueLogFileSize, "valuelogfilesize",
+		"Specify the datastore value log file size (in bytes). In memory size will be 2*valuelogfilesize",
+	)
+	err = viper.BindPFlag("datastore.badger.valuelogfilesize", startCmd.Flags().Lookup("valuelogfilesize"))
+	if err != nil {
+		log.FeedbackFatalE(context.Background(), "Could not bind datastore.badger.valuelogfilesize", err)
+	}
+
 	startCmd.Flags().String(
 		"p2paddr", cfg.Net.P2PAddress,
 		"Listener address for the p2p network (formatted as a libp2p MultiAddr)",

config/config.go

@@ -50,10 +50,13 @@ import (
 	"fmt"
 	"net"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
+	"unicode"
 
+	"github.com/mitchellh/mapstructure"
 	ma "github.com/multiformats/go-multiaddr"
 	badgerds "github.com/sourcenetwork/defradb/datastore/badger/v3"
 	"github.com/sourcenetwork/defradb/logging"
@@ -94,11 +97,13 @@ func (cfg *Config) Load(rootDirPath string) error {
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.AutomaticEnv()
 
-	if err := viper.Unmarshal(cfg); err != nil {
+	err := viper.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc()))
+	if err != nil {
 		return err
 	}
+
 	cfg.handleParams(rootDirPath)
-	err := cfg.validate()
+	err = cfg.validate()
 	if err != nil {
 		return err
 	}
@@ -124,13 +129,15 @@ func (cfg *Config) LoadWithoutRootDir() error {
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.AutomaticEnv()
 
-	if err := viper.Unmarshal(cfg); err != nil {
+	err = viper.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc()))
+	if err != nil {
 		return err
 	}
 	rootDir, err := DefaultRootDir()
 	if err != nil {
 		log.FatalE(context.Background(), "Could not get home directory", err)
 	}
+
 	cfg.handleParams(rootDir)
 	err = cfg.validate()
 	if err != nil {
@@ -170,6 +177,11 @@ func (cfg *Config) handleParams(rootDir string) {
 	if !filepath.IsAbs(cfg.Datastore.Badger.Path) {
 		cfg.Datastore.Badger.Path = filepath.Join(rootDir, cfg.Datastore.Badger.Path)
 	}
+	cfg.setBadgerVLogMaxSize()
+}
+
+func (cfg *Config) setBadgerVLogMaxSize() {
+	cfg.Datastore.Badger.Options.ValueLogFileSize = int64(cfg.Datastore.Badger.ValueLogFileSize)
 }
 
 // DatastoreConfig configures datastores.
@@ -181,20 +193,97 @@ type DatastoreConfig struct {
 
 // BadgerConfig configures Badger's on-disk / filesystem mode.
 type BadgerConfig struct {
-	Path string
+	Path             string
+	ValueLogFileSize ByteSize
 	*badgerds.Options
 }
 
+type ByteSize uint64
+
+const (
+	B   ByteSize = 1
+	KiB          = B << 10
+	MiB          = KiB << 10
+	GiB          = MiB << 10
+	TiB          = GiB << 10
+	PiB          = TiB << 10
+)
+
+// UnmarshalText calls Set on ByteSize with the given text
+func (bs *ByteSize) UnmarshalText(text []byte) error {
+	return bs.Set(string(text))
+}
+
+// Set parses a string into ByteSize
+func (bs *ByteSize) Set(s string) error {
+	digitString := ""
+	unit := ""
+	for _, char := range s {
+		if unicode.IsDigit(char) {
+			digitString += string(char)
+		} else {
+			unit += string(char)
+		}
+	}
+	digits, err := strconv.Atoi(digitString)
+	if err != nil {
+		return err
+	}
+
+	switch strings.ToUpper(strings.Trim(unit, " ")) {
+	case "B":
+		*bs = ByteSize(digits) * B
+	case "KB", "KIB":
+		*bs = ByteSize(digits) * KiB
+	case "MB", "MIB":
+		*bs = ByteSize(digits) * MiB
+	case "GB", "GIB":
+		*bs = ByteSize(digits) * GiB
+	case "TB", "TIB":
+		*bs = ByteSize(digits) * TiB
+	case "PB", "PIB":
+		*bs = ByteSize(digits) * PiB
+	default:
+		*bs = ByteSize(digits)
+	}
+
+	return nil
+}
+
+// String returns the string formatted output of ByteSize
+func (bs *ByteSize) String() string {
+	const unit = 1024
+	bsInt := int64(*bs)
+	if bsInt < unit {
+		return fmt.Sprintf("%d", bsInt)
+	}
+	div, exp := int64(unit), 0
+	for n := bsInt / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%d%ciB", bsInt/div, "KMGTP"[exp])
+}
+
+// Type returns the type as a string.
+func (bs *ByteSize) Type() string {
+	return "ByteSize"
+}
+
 // MemoryConfig configures of Badger's memory mode.
 type MemoryConfig struct {
 	Size uint64
 }
 
 func defaultDatastoreConfig() *DatastoreConfig {
+	// create a copy of the default badger options
+	opts := badgerds.DefaultOptions
 	return &DatastoreConfig{
 		Store: "badger",
 		Badger: BadgerConfig{
-			Path: "data",
+			Path:             "data",
+			ValueLogFileSize: 1 * GiB,
+			Options:          &opts,
 		},
 	}
 }

config/config_test.go

@@ -341,3 +341,122 @@ func TestNodeConfig(t *testing.T) {
 	assert.Equal(t, expectedOptions.EnablePubSub, options.EnablePubSub)
 	assert.Equal(t, expectedOptions.EnableRelay, options.EnableRelay)
 }
+
+func TestUnmarshalByteSize(t *testing.T) {
+	var bs ByteSize
+
+	b := []byte("10")
+	err := bs.UnmarshalText(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*B, bs)
+
+	b = []byte("10B")
+	err = bs.UnmarshalText(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*B, bs)
+
+	b = []byte("10 B")
+	err = bs.UnmarshalText(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*B, bs)
+
+	kb := []byte("10KB")
+	err = bs.UnmarshalText(kb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*KiB, bs)
+
+	kb = []byte("10KiB")
+	err = bs.UnmarshalText(kb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*KiB, bs)
+
+	kb = []byte("10 kb")
+	err = bs.UnmarshalText(kb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*KiB, bs)
+
+	mb := []byte("10MB")
+	err = bs.UnmarshalText(mb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*MiB, bs)
+
+	mb = []byte("10MiB")
+	err = bs.UnmarshalText(mb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*MiB, bs)
+
+	gb := []byte("10GB")
+	err = bs.UnmarshalText(gb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*GiB, bs)
+
+	gb = []byte("10GiB")
+	err = bs.UnmarshalText(gb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*GiB, bs)
+
+	tb := []byte("10TB")
+	err = bs.UnmarshalText(tb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*TiB, bs)
+
+	tb = []byte("10TiB")
+	err = bs.UnmarshalText(tb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*TiB, bs)
+
+	pb := []byte("10PB")
+	err = bs.UnmarshalText(pb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*PiB, bs)
+
+	pb = []byte("10PiB")
+	err = bs.UnmarshalText(pb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, 10*PiB, bs)
+
+	eb := []byte("१")
+	err = bs.UnmarshalText(eb)
+	assert.Error(t, err)
+}
+
+func TestByteSizeType(t *testing.T) {
+	var bs ByteSize
+	assert.Equal(t, "ByteSize", bs.Type())
+}
+
+func TestByteSizeToString(t *testing.T) {
+	b := 999 * B
+	assert.Equal(t, "999", b.String())
+
+	mb := 10 * MiB
+	assert.Equal(t, "10MiB", mb.String())
+}

config/configfile.go

@@ -54,6 +54,9 @@ datastore:
   store: {{ .Datastore.Store }}
   badger:
     path: {{ .Datastore.Badger.Path }}
+    # Maximum file size of the value log files. The in-memory file size will be 2*valuelogfilesize.
+    # Human friendly units can be used (ex: 500MB).
+    valuelogfilesize: {{ .Datastore.Badger.ValueLogFileSize }}
   # memory:
   #    size: {{ .Datastore.Memory.Size }}
 

config/configfile_test.go

@@ -102,3 +102,34 @@ func TestReadConfigFileForLogger(t *testing.T) {
 	assert.Equal(t, cfg.Log.Output, cfgFromFile.Log.Output)
 	assert.Equal(t, cfg.Log.Stacktrace, cfgFromFile.Log.Stacktrace)
 }
+
+func TestReadConfigFileForDatastore(t *testing.T) {
+	dir := t.TempDir()
+
+	cfg := DefaultConfig()
+	cfg.Datastore.Store = "badger"
+	cfg.Datastore.Badger.Path = "dataPath"
+	cfg.Datastore.Badger.ValueLogFileSize = 512 * MiB
+
+	err := cfg.WriteConfigFileToRootDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	path := dir + "/" + DefaultDefraDBConfigFileName
+
+	_, err = os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cfgFromFile := DefaultConfig()
+
+	err = cfgFromFile.Load(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, cfg.Datastore.Store, cfgFromFile.Datastore.Store)
+	assert.Equal(t, dir+"/"+cfg.Datastore.Badger.Path, cfgFromFile.Datastore.Badger.Path)
+	assert.Equal(t, cfg.Datastore.Badger.ValueLogFileSize, cfgFromFile.Datastore.Badger.ValueLogFileSize)
+}