From 32f1f589345facf64493c68b3f9dc534ab936650 Mon Sep 17 00:00:00 2001 From: Rohan Prasad Date: Thu, 1 Apr 2021 22:52:41 +0530 Subject: [PATCH] feat(flags): [Breaking] Add cache superflag for alpha (#7652) This change introduces cache as a superflag, with size-mb and percentage as subflags for dgraph alphas. --- dgraph/cmd/alpha/admin.go | 2 +- dgraph/cmd/alpha/run.go | 32 ++++++++++++++++---------------- worker/config.go | 2 -- worker/server_state.go | 1 + worker/worker.go | 4 +++- x/flags.go | 3 --- 6 files changed, 21 insertions(+), 23 deletions(-) diff --git a/dgraph/cmd/alpha/admin.go b/dgraph/cmd/alpha/admin.go index d43fb285b3d..32f498853d4 100644 --- a/dgraph/cmd/alpha/admin.go +++ b/dgraph/cmd/alpha/admin.go @@ -222,7 +222,7 @@ func memoryLimitPutHandler(w http.ResponseWriter, r *http.Request) { } gqlReq := &schema.Request{ Query: ` - mutation config($cacheMb: Int) { + mutation config($cacheMb: Float) { config(input: {cacheMb: $cacheMb}) { response { code diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index 538b773f8ca..d3ff19d22aa 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -108,10 +108,6 @@ they form a Raft group and provide synchronous replication. // --encryption_key_file enc.RegisterFlags(flag) - flag.String("cache_percentage", "0,65,35,0", - `Cache percentages summing up to 100 for various caches (FORMAT: PostingListCache,`+ - `PstoreBlockCache,PstoreIndexCache,WAL).`) - flag.StringP("postings", "p", "p", "Directory to store posting lists.") flag.String("tmp", "t", "Directory to store temporary buffers.") @@ -149,6 +145,16 @@ they form a Raft group and provide synchronous replication. "worker in a failed state. Use -1 to retry infinitely."). String()) + // Cache flags. + flag.String("cache", worker.CacheDefaults, z.NewSuperFlagHelp(worker.CacheDefaults). + Head("Cache options"). + Flag("size-mb", + "Total size of cache (in MB) to be used in Dgraph."). + Flag("percentage", + "Cache percentages summing up to 100 for various caches (FORMAT: PostingListCache,"+ + "PstoreBlockCache,PstoreIndexCache)"). + String()) + flag.String("raft", worker.RaftDefaults, z.NewSuperFlagHelp(worker.RaftDefaults). Head("Raft options"). Flag("idx", @@ -618,23 +624,17 @@ func run() { } bindall = Alpha.Conf.GetBool("bindall") - - totalCache := int64(Alpha.Conf.GetInt("cache_mb")) + cache := z.NewSuperFlag(Alpha.Conf.GetString("cache")).MergeAndCheckDefault( + worker.CacheDefaults) + totalCache := cache.GetInt64("size-mb") x.AssertTruef(totalCache >= 0, "ERROR: Cache size must be non-negative") - if Alpha.Conf.IsSet("lru_mb") { - glog.Warningln("--lru_mb is deprecated, use --cache_mb instead") - if !Alpha.Conf.IsSet("cache_mb") { - totalCache = int64(Alpha.Conf.GetFloat64("lru_mb")) - } - } - cachePercentage := Alpha.Conf.GetString("cache_percentage") - cachePercent, err := x.GetCachePercentages(cachePercentage, 4) + cachePercentage := cache.GetString("percentage") + cachePercent, err := x.GetCachePercentages(cachePercentage, 3) x.Check(err) postingListCacheSize := (cachePercent[0] * (totalCache << 20)) / 100 pstoreBlockCacheSize := (cachePercent[1] * (totalCache << 20)) / 100 pstoreIndexCacheSize := (cachePercent[2] * (totalCache << 20)) / 100 - walCache := (cachePercent[3] * (totalCache << 20)) / 100 badger := z.NewSuperFlag(Alpha.Conf.GetString("badger")).MergeAndCheckDefault( worker.BadgerDefaults) @@ -648,10 +648,10 @@ func run() { WALDir: Alpha.Conf.GetString("wal"), PostingDirCompression: ctype, PostingDirCompressionLevel: clevel, + CacheMb: totalCache, CachePercentage: cachePercentage, PBlockCacheSize: pstoreBlockCacheSize, PIndexCacheSize: pstoreIndexCacheSize, - WalCache: walCache, MutationsMode: worker.AllowMutations, AuthToken: security.GetString("token"), diff --git a/worker/config.go b/worker/config.go index 7d520a6bd08..0daae39e376 100644 --- a/worker/config.go +++ b/worker/config.go @@ -54,8 +54,6 @@ type Options struct { PBlockCacheSize int64 // PIndexCacheSize is the size of index cache for pstore PIndexCacheSize int64 - // WalCache is the size of block cache for wstore - WalCache int64 // HmacSecret stores the secret used to sign JSON Web Tokens (JWT). HmacSecret x.SensitiveByteSlice diff --git a/worker/server_state.go b/worker/server_state.go index 1f9d7520b80..27019afeefc 100644 --- a/worker/server_state.go +++ b/worker/server_state.go @@ -52,6 +52,7 @@ const ( ZeroLimitsDefaults = `uid-lease=0; refill-interval=30s` GraphQLDefaults = `introspection=true; debug=false; extensions=true; poll-interval=1s; ` + `lambda-url=;` + CacheDefaults = `size-mb=1024; percentage=0,65,35;` ) // ServerState holds the state of the Dgraph server. diff --git a/worker/worker.go b/worker/worker.go index 87a2c31f3b8..2be3103d95d 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -149,7 +149,7 @@ func UpdateCacheMb(memoryMB int64) error { return errors.Errorf("cache_mb must be non-negative") } - cachePercent, err := x.GetCachePercentages(Config.CachePercentage, 4) + cachePercent, err := x.GetCachePercentages(Config.CachePercentage, 3) if err != nil { return err } @@ -164,6 +164,8 @@ func UpdateCacheMb(memoryMB int64) error { if _, err := pstore.CacheMaxCost(badger.IndexCache, indexCacheSize); err != nil { return errors.Wrapf(err, "cannot update index cache size") } + + Config.CacheMb = memoryMB return nil } diff --git a/x/flags.go b/x/flags.go index 13ea3b71cfb..198327dad02 100644 --- a/x/flags.go +++ b/x/flags.go @@ -54,9 +54,6 @@ func FillCommonFlags(flag *pflag.FlagSet) { `guaranteeing no data loss in case of hard reboot.`+"\n "+ `Most users should be OK with choosing "process".`) - // Cache flags. - flag.Int64("cache_mb", 1024, "Total size of cache (in MB) to be used in Dgraph.") - flag.String("telemetry", TelemetryDefaults, z.NewSuperFlagHelp(TelemetryDefaults). Head("Telemetry (diagnostic) options"). Flag("reports",