From 2a741fb860baaa18c73fb7953befe613637068c8 Mon Sep 17 00:00:00 2001 From: Fred Carle Date: Mon, 15 Aug 2022 23:34:54 -0400 Subject: [PATCH 1/3] add vlogmaxsize config option --- cli/start.go | 9 ++++ config/config.go | 94 +++++++++++++++++++++++++++++++++++++-- config/config_test.go | 84 ++++++++++++++++++++++++++++++++++ config/configfile.go | 3 ++ config/configfile_test.go | 31 +++++++++++++ 5 files changed, 217 insertions(+), 4 deletions(-) diff --git a/cli/start.go b/cli/start.go index 108c5ea09e..0b132a1f01 100644 --- a/cli/start.go +++ b/cli/start.go @@ -235,6 +235,15 @@ func init() { log.FeedbackFatalE(context.Background(), "Could not bind datastore.store", err) } + startCmd.Flags().Var( + &cfg.Datastore.Badger.VLogMaxSize, "vlogmaxsize", + "Specify the datastore maximum value log file size (in bytes)", + ) + err = viper.BindPFlag("datastore.badger.vlogmaxsize", startCmd.Flags().Lookup("vlogmaxsize")) + if err != nil { + log.FeedbackFatalE(context.Background(), "Could not bind datastore..badger.vlogmaxsize", err) + } + startCmd.Flags().String( "p2paddr", cfg.Net.P2PAddress, "Listener address for the p2p network (formatted as a libp2p MultiAddr)", diff --git a/config/config.go b/config/config.go index 336c480c27..ef554424f5 100644 --- a/config/config.go +++ b/config/config.go @@ -50,10 +50,13 @@ import ( "fmt" "net" "path/filepath" + "strconv" "strings" "text/template" "time" + "unicode" + "github.com/mitchellh/mapstructure" ma "github.com/multiformats/go-multiaddr" badgerds "github.com/sourcenetwork/defradb/datastore/badger/v3" "github.com/sourcenetwork/defradb/logging" @@ -94,9 +97,10 @@ func (cfg *Config) Load(rootDirPath string) error { viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.AutomaticEnv() - if err := viper.Unmarshal(cfg); err != nil { + if err := viper.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())); err != nil { return err } + cfg.setBadgerVLogMaxSize() cfg.handleParams(rootDirPath) err := cfg.validate() if err != nil { @@ -124,13 +128,14 @@ func (cfg *Config) LoadWithoutRootDir() error { viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.AutomaticEnv() - if err := viper.Unmarshal(cfg); err != nil { + if err := viper.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())); err != nil { return err } rootDir, err := DefaultRootDir() if err != nil { log.FatalE(context.Background(), "Could not get home directory", err) } + cfg.setBadgerVLogMaxSize() cfg.handleParams(rootDir) err = cfg.validate() if err != nil { @@ -172,6 +177,10 @@ func (cfg *Config) handleParams(rootDir string) { } } +func (cfg *Config) setBadgerVLogMaxSize() { + cfg.Datastore.Badger.ValueLogFileSize = int64(cfg.Datastore.Badger.VLogMaxSize) +} + // DatastoreConfig configures datastores. type DatastoreConfig struct { Store string @@ -181,20 +190,97 @@ type DatastoreConfig struct { // BadgerConfig configures Badger's on-disk / filesystem mode. type BadgerConfig struct { - Path string + Path string + VLogMaxSize ByteSize *badgerds.Options } +type ByteSize uint64 + +const ( + B ByteSize = 1 + KB = B << 10 + MB = KB << 10 + GB = MB << 10 + TB = GB << 10 + PB = TB << 10 +) + +// UnmarshallText calls Set on ByteSize with the given text +func (bs *ByteSize) UnmarshalText(text []byte) error { + return bs.Set(string(text)) +} + +// Set parses a string into ByteSize +func (bs *ByteSize) Set(s string) error { + digitString := "" + unit := "" + for _, char := range s { + if unicode.IsDigit(char) { + digitString += string(char) + } else { + unit += string(char) + } + } + digits, err := strconv.Atoi(digitString) + if err != nil { + return err + } + + switch strings.ToUpper(strings.Trim(unit, " ")) { + case "B": + *bs = ByteSize(digits) * B + case "KB": + *bs = ByteSize(digits) * KB + case "MB": + *bs = ByteSize(digits) * MB + case "GB": + *bs = ByteSize(digits) * GB + case "TB": + *bs = ByteSize(digits) * TB + case "PB": + *bs = ByteSize(digits) * PB + default: + *bs = ByteSize(digits) + } + + return nil +} + +// String returns the string formatted output of ByteSize +func (bs *ByteSize) String() string { + const unit = 1000 + bsInt := int64(*bs) + if bsInt < unit { + return fmt.Sprintf("%d", bsInt) + } + div, exp := int64(unit), 0 + for n := bsInt / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%d%cB", bsInt/div, "KMGTP"[exp]) +} + +// Type returns the type as a string. +func (bs *ByteSize) Type() string { + return "ByteSize" +} + // MemoryConfig configures of Badger's memory mode. type MemoryConfig struct { Size uint64 } func defaultDatastoreConfig() *DatastoreConfig { + // create a copy of the default badger options + opts := badgerds.DefaultOptions return &DatastoreConfig{ Store: "badger", Badger: BadgerConfig{ - Path: "data", + Path: "data", + VLogMaxSize: 1 * GB, + Options: &opts, }, } } diff --git a/config/config_test.go b/config/config_test.go index 98aaee8bec..125ca16af8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -341,3 +341,87 @@ func TestNodeConfig(t *testing.T) { assert.Equal(t, expectedOptions.EnablePubSub, options.EnablePubSub) assert.Equal(t, expectedOptions.EnableRelay, options.EnableRelay) } + +func TestUnmarshallByteSize(t *testing.T) { + var bs ByteSize + + b := []byte("10") + err := bs.UnmarshalText(b) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*B, bs) + + b = []byte("10B") + err = bs.UnmarshalText(b) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*B, bs) + + b = []byte("10 B") + err = bs.UnmarshalText(b) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*B, bs) + + kb := []byte("10KB") + err = bs.UnmarshalText(kb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*KB, bs) + + kb = []byte("10 kb") + err = bs.UnmarshalText(kb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*KB, bs) + + mb := []byte("10MB") + err = bs.UnmarshalText(mb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*MB, bs) + + gb := []byte("10GB") + err = bs.UnmarshalText(gb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*GB, bs) + + tb := []byte("10TB") + err = bs.UnmarshalText(tb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*TB, bs) + + pb := []byte("10PB") + err = bs.UnmarshalText(pb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*PB, bs) + + eb := []byte("१") + err = bs.UnmarshalText(eb) + assert.Error(t, err) +} + +func TestByteSizeType(t *testing.T) { + var bs ByteSize + assert.Equal(t, "ByteSize", bs.Type()) +} + +func TestByteSizeToString(t *testing.T) { + b := 999 * B + assert.Equal(t, "999", b.String()) + + mb := 10 * MB + assert.Equal(t, "10MB", mb.String()) +} diff --git a/config/configfile.go b/config/configfile.go index 4dfb048b8c..a30c0437eb 100644 --- a/config/configfile.go +++ b/config/configfile.go @@ -54,6 +54,9 @@ datastore: store: {{ .Datastore.Store }} badger: path: {{ .Datastore.Badger.Path }} + # Maximum file size of the value log file. The actual file size will be 2*vlogmaxsize. + # Human friendly units can be used (ex: 500MB). + vlogmaxsize: {{ .Datastore.Badger.VLogMaxSize }} # memory: # size: {{ .Datastore.Memory.Size }} diff --git a/config/configfile_test.go b/config/configfile_test.go index 913a43c6e2..a14e695fb4 100644 --- a/config/configfile_test.go +++ b/config/configfile_test.go @@ -102,3 +102,34 @@ func TestReadConfigFileForLogger(t *testing.T) { assert.Equal(t, cfg.Log.Output, cfgFromFile.Log.Output) assert.Equal(t, cfg.Log.Stacktrace, cfgFromFile.Log.Stacktrace) } + +func TestReadConfigFileForDatastore(t *testing.T) { + dir := t.TempDir() + + cfg := DefaultConfig() + cfg.Datastore.Store = "badger" + cfg.Datastore.Badger.Path = "dataPath" + cfg.Datastore.Badger.VLogMaxSize = 512 * MB + + err := cfg.WriteConfigFileToRootDir(dir) + if err != nil { + t.Fatal(err) + } + + path := dir + "/" + DefaultDefraDBConfigFileName + + _, err = os.Stat(path) + if err != nil { + t.Fatal(err) + } + + cfgFromFile := DefaultConfig() + + err = cfgFromFile.Load(dir) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, cfg.Datastore.Store, cfgFromFile.Datastore.Store) + assert.Equal(t, dir+"/"+cfg.Datastore.Badger.Path, cfgFromFile.Datastore.Badger.Path) + assert.NotEqual(t, cfg.Datastore.Badger.VLogMaxSize, cfgFromFile.Datastore.Badger.VLogMaxSize) +} From 89724d021153f07e1683f87ac2d44fa5ae9f5b64 Mon Sep 17 00:00:00 2001 From: Fred Carle Date: Tue, 16 Aug 2022 18:44:22 -0400 Subject: [PATCH 2/3] apply feedback --- cli/start.go | 8 ++++---- config/config.go | 20 +++++++++++--------- config/configfile.go | 4 ++-- config/configfile_test.go | 4 ++-- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/cli/start.go b/cli/start.go index 0b132a1f01..c3ea17568b 100644 --- a/cli/start.go +++ b/cli/start.go @@ -236,12 +236,12 @@ func init() { } startCmd.Flags().Var( - &cfg.Datastore.Badger.VLogMaxSize, "vlogmaxsize", - "Specify the datastore maximum value log file size (in bytes)", + &cfg.Datastore.Badger.ValueLogFileSize, "valuelogfilesize", + "Specify the datastore value log file size (in bytes). In memory size will be 2*valuelogfilesize", ) - err = viper.BindPFlag("datastore.badger.vlogmaxsize", startCmd.Flags().Lookup("vlogmaxsize")) + err = viper.BindPFlag("datastore.badger.valuelogfilesize", startCmd.Flags().Lookup("valuelogfilesize")) if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind datastore..badger.vlogmaxsize", err) + log.FeedbackFatalE(context.Background(), "Could not bind datastore.badger.valuelogfilesize", err) } startCmd.Flags().String( diff --git a/config/config.go b/config/config.go index ef554424f5..d8c3d615e6 100644 --- a/config/config.go +++ b/config/config.go @@ -97,12 +97,13 @@ func (cfg *Config) Load(rootDirPath string) error { viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.AutomaticEnv() - if err := viper.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())); err != nil { + err := viper.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())) + if err != nil { return err } cfg.setBadgerVLogMaxSize() cfg.handleParams(rootDirPath) - err := cfg.validate() + err = cfg.validate() if err != nil { return err } @@ -128,7 +129,8 @@ func (cfg *Config) LoadWithoutRootDir() error { viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.AutomaticEnv() - if err := viper.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())); err != nil { + err = viper.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())) + if err != nil { return err } rootDir, err := DefaultRootDir() @@ -178,7 +180,7 @@ func (cfg *Config) handleParams(rootDir string) { } func (cfg *Config) setBadgerVLogMaxSize() { - cfg.Datastore.Badger.ValueLogFileSize = int64(cfg.Datastore.Badger.VLogMaxSize) + cfg.Datastore.Badger.Options.ValueLogFileSize = int64(cfg.Datastore.Badger.ValueLogFileSize) } // DatastoreConfig configures datastores. @@ -190,8 +192,8 @@ type DatastoreConfig struct { // BadgerConfig configures Badger's on-disk / filesystem mode. type BadgerConfig struct { - Path string - VLogMaxSize ByteSize + Path string + ValueLogFileSize ByteSize *badgerds.Options } @@ -278,9 +280,9 @@ func defaultDatastoreConfig() *DatastoreConfig { return &DatastoreConfig{ Store: "badger", Badger: BadgerConfig{ - Path: "data", - VLogMaxSize: 1 * GB, - Options: &opts, + Path: "data", + ValueLogFileSize: 1 * GB, + Options: &opts, }, } } diff --git a/config/configfile.go b/config/configfile.go index a30c0437eb..ee2f33c7b3 100644 --- a/config/configfile.go +++ b/config/configfile.go @@ -54,9 +54,9 @@ datastore: store: {{ .Datastore.Store }} badger: path: {{ .Datastore.Badger.Path }} - # Maximum file size of the value log file. The actual file size will be 2*vlogmaxsize. + # Maximum file size of the value log files. The in memory file size will be 2*valuelogfilesize. # Human friendly units can be used (ex: 500MB). - vlogmaxsize: {{ .Datastore.Badger.VLogMaxSize }} + valuelogfilesize: {{ .Datastore.Badger.ValueLogFileSize }} # memory: # size: {{ .Datastore.Memory.Size }} diff --git a/config/configfile_test.go b/config/configfile_test.go index a14e695fb4..5d1e1b36a3 100644 --- a/config/configfile_test.go +++ b/config/configfile_test.go @@ -109,7 +109,7 @@ func TestReadConfigFileForDatastore(t *testing.T) { cfg := DefaultConfig() cfg.Datastore.Store = "badger" cfg.Datastore.Badger.Path = "dataPath" - cfg.Datastore.Badger.VLogMaxSize = 512 * MB + cfg.Datastore.Badger.ValueLogFileSize = 512 * MB err := cfg.WriteConfigFileToRootDir(dir) if err != nil { @@ -131,5 +131,5 @@ func TestReadConfigFileForDatastore(t *testing.T) { } assert.Equal(t, cfg.Datastore.Store, cfgFromFile.Datastore.Store) assert.Equal(t, dir+"/"+cfg.Datastore.Badger.Path, cfgFromFile.Datastore.Badger.Path) - assert.NotEqual(t, cfg.Datastore.Badger.VLogMaxSize, cfgFromFile.Datastore.Badger.VLogMaxSize) + assert.NotEqual(t, cfg.Datastore.Badger.ValueLogFileSize, cfgFromFile.Datastore.Badger.ValueLogFileSize) } From cf2a0a156d8a541bd2b8e0c3553bfb850169b0c3 Mon Sep 17 00:00:00 2001 From: Fred Carle Date: Wed, 17 Aug 2022 14:30:06 -0400 Subject: [PATCH 3/3] apply feedback --- config/config.go | 45 +++++++++++++++++---------------- config/config_test.go | 53 ++++++++++++++++++++++++++++++++------- config/configfile.go | 2 +- config/configfile_test.go | 4 +-- 4 files changed, 70 insertions(+), 34 deletions(-) diff --git a/config/config.go b/config/config.go index d8c3d615e6..f4de2acfc2 100644 --- a/config/config.go +++ b/config/config.go @@ -101,7 +101,7 @@ func (cfg *Config) Load(rootDirPath string) error { if err != nil { return err } - cfg.setBadgerVLogMaxSize() + cfg.handleParams(rootDirPath) err = cfg.validate() if err != nil { @@ -137,7 +137,7 @@ func (cfg *Config) LoadWithoutRootDir() error { if err != nil { log.FatalE(context.Background(), "Could not get home directory", err) } - cfg.setBadgerVLogMaxSize() + cfg.handleParams(rootDir) err = cfg.validate() if err != nil { @@ -177,6 +177,7 @@ func (cfg *Config) handleParams(rootDir string) { if !filepath.IsAbs(cfg.Datastore.Badger.Path) { cfg.Datastore.Badger.Path = filepath.Join(rootDir, cfg.Datastore.Badger.Path) } + cfg.setBadgerVLogMaxSize() } func (cfg *Config) setBadgerVLogMaxSize() { @@ -200,15 +201,15 @@ type BadgerConfig struct { type ByteSize uint64 const ( - B ByteSize = 1 - KB = B << 10 - MB = KB << 10 - GB = MB << 10 - TB = GB << 10 - PB = TB << 10 + B ByteSize = 1 + KiB = B << 10 + MiB = KiB << 10 + GiB = MiB << 10 + TiB = GiB << 10 + PiB = TiB << 10 ) -// UnmarshallText calls Set on ByteSize with the given text +// UnmarshalText calls Set on ByteSize with the given text func (bs *ByteSize) UnmarshalText(text []byte) error { return bs.Set(string(text)) } @@ -232,16 +233,16 @@ func (bs *ByteSize) Set(s string) error { switch strings.ToUpper(strings.Trim(unit, " ")) { case "B": *bs = ByteSize(digits) * B - case "KB": - *bs = ByteSize(digits) * KB - case "MB": - *bs = ByteSize(digits) * MB - case "GB": - *bs = ByteSize(digits) * GB - case "TB": - *bs = ByteSize(digits) * TB - case "PB": - *bs = ByteSize(digits) * PB + case "KB", "KIB": + *bs = ByteSize(digits) * KiB + case "MB", "MIB": + *bs = ByteSize(digits) * MiB + case "GB", "GIB": + *bs = ByteSize(digits) * GiB + case "TB", "TIB": + *bs = ByteSize(digits) * TiB + case "PB", "PIB": + *bs = ByteSize(digits) * PiB default: *bs = ByteSize(digits) } @@ -251,7 +252,7 @@ func (bs *ByteSize) Set(s string) error { // String returns the string formatted output of ByteSize func (bs *ByteSize) String() string { - const unit = 1000 + const unit = 1024 bsInt := int64(*bs) if bsInt < unit { return fmt.Sprintf("%d", bsInt) @@ -261,7 +262,7 @@ func (bs *ByteSize) String() string { div *= unit exp++ } - return fmt.Sprintf("%d%cB", bsInt/div, "KMGTP"[exp]) + return fmt.Sprintf("%d%ciB", bsInt/div, "KMGTP"[exp]) } // Type returns the type as a string. @@ -281,7 +282,7 @@ func defaultDatastoreConfig() *DatastoreConfig { Store: "badger", Badger: BadgerConfig{ Path: "data", - ValueLogFileSize: 1 * GB, + ValueLogFileSize: 1 * GiB, Options: &opts, }, } diff --git a/config/config_test.go b/config/config_test.go index 125ca16af8..b2b584afc0 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -342,7 +342,7 @@ func TestNodeConfig(t *testing.T) { assert.Equal(t, expectedOptions.EnableRelay, options.EnableRelay) } -func TestUnmarshallByteSize(t *testing.T) { +func TestUnmarshalByteSize(t *testing.T) { var bs ByteSize b := []byte("10") @@ -371,42 +371,77 @@ func TestUnmarshallByteSize(t *testing.T) { if err != nil { t.Fatal(err) } - assert.Equal(t, 10*KB, bs) + assert.Equal(t, 10*KiB, bs) + + kb = []byte("10KiB") + err = bs.UnmarshalText(kb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*KiB, bs) kb = []byte("10 kb") err = bs.UnmarshalText(kb) if err != nil { t.Fatal(err) } - assert.Equal(t, 10*KB, bs) + assert.Equal(t, 10*KiB, bs) mb := []byte("10MB") err = bs.UnmarshalText(mb) if err != nil { t.Fatal(err) } - assert.Equal(t, 10*MB, bs) + assert.Equal(t, 10*MiB, bs) + + mb = []byte("10MiB") + err = bs.UnmarshalText(mb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*MiB, bs) gb := []byte("10GB") err = bs.UnmarshalText(gb) if err != nil { t.Fatal(err) } - assert.Equal(t, 10*GB, bs) + assert.Equal(t, 10*GiB, bs) + + gb = []byte("10GiB") + err = bs.UnmarshalText(gb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*GiB, bs) tb := []byte("10TB") err = bs.UnmarshalText(tb) if err != nil { t.Fatal(err) } - assert.Equal(t, 10*TB, bs) + assert.Equal(t, 10*TiB, bs) + + tb = []byte("10TiB") + err = bs.UnmarshalText(tb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*TiB, bs) pb := []byte("10PB") err = bs.UnmarshalText(pb) if err != nil { t.Fatal(err) } - assert.Equal(t, 10*PB, bs) + assert.Equal(t, 10*PiB, bs) + + pb = []byte("10PiB") + err = bs.UnmarshalText(pb) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 10*PiB, bs) eb := []byte("१") err = bs.UnmarshalText(eb) @@ -422,6 +457,6 @@ func TestByteSizeToString(t *testing.T) { b := 999 * B assert.Equal(t, "999", b.String()) - mb := 10 * MB - assert.Equal(t, "10MB", mb.String()) + mb := 10 * MiB + assert.Equal(t, "10MiB", mb.String()) } diff --git a/config/configfile.go b/config/configfile.go index ee2f33c7b3..9d1a2b9cf5 100644 --- a/config/configfile.go +++ b/config/configfile.go @@ -54,7 +54,7 @@ datastore: store: {{ .Datastore.Store }} badger: path: {{ .Datastore.Badger.Path }} - # Maximum file size of the value log files. The in memory file size will be 2*valuelogfilesize. + # Maximum file size of the value log files. The in-memory file size will be 2*valuelogfilesize. # Human friendly units can be used (ex: 500MB). valuelogfilesize: {{ .Datastore.Badger.ValueLogFileSize }} # memory: diff --git a/config/configfile_test.go b/config/configfile_test.go index 5d1e1b36a3..5080964966 100644 --- a/config/configfile_test.go +++ b/config/configfile_test.go @@ -109,7 +109,7 @@ func TestReadConfigFileForDatastore(t *testing.T) { cfg := DefaultConfig() cfg.Datastore.Store = "badger" cfg.Datastore.Badger.Path = "dataPath" - cfg.Datastore.Badger.ValueLogFileSize = 512 * MB + cfg.Datastore.Badger.ValueLogFileSize = 512 * MiB err := cfg.WriteConfigFileToRootDir(dir) if err != nil { @@ -131,5 +131,5 @@ func TestReadConfigFileForDatastore(t *testing.T) { } assert.Equal(t, cfg.Datastore.Store, cfgFromFile.Datastore.Store) assert.Equal(t, dir+"/"+cfg.Datastore.Badger.Path, cfgFromFile.Datastore.Badger.Path) - assert.NotEqual(t, cfg.Datastore.Badger.ValueLogFileSize, cfgFromFile.Datastore.Badger.ValueLogFileSize) + assert.Equal(t, cfg.Datastore.Badger.ValueLogFileSize, cfgFromFile.Datastore.Badger.ValueLogFileSize) }