allow setting a custom 'GOGC' value when node is not ready

cluster/config.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"net"
 	"net/http"
+	"os"
+	"strconv"
 	"time"
 
 	"github.com/grafana/globalconf"
@@ -19,6 +21,8 @@ var (
 	maxPrio            int
 	httpTimeout        time.Duration
 	minAvailableShards int
+	gcPercent          int
+	gcPercentNotReady  int
 
 	swimUseConfig               = "default-lan"
 	swimAdvertiseAddrStr        string
@@ -46,6 +50,22 @@ var (
 )
 
 func ConfigSetup() {
+
+	// behavior here is same as standard runtime:
+	// unparseable or not set -> 100
+	// "off" -> -1
+	gcPercent = 100
+	gogc := os.Getenv("GOGC")
+	if gogc != "" {
+		if gogc == "off" {
+			gcPercent = -1
+		}
+		val, err := strconv.Atoi(gogc)
+		if err == nil {
+			gcPercent = val
+		}
+	}
+
 	clusterCfg := flag.NewFlagSet("cluster", flag.ExitOnError)
 	clusterCfg.StringVar(&ClusterName, "name", "metrictank", "Unique name of the cluster.")
 	clusterCfg.BoolVar(&primary, "primary-node", false, "the primary node writes data to cassandra. There should only be 1 primary node per shardGroup.")
@@ -54,6 +74,7 @@ func ConfigSetup() {
 	clusterCfg.DurationVar(&httpTimeout, "http-timeout", time.Second*60, "How long to wait before aborting http requests to cluster peers and returning a http 503 service unavailable")
 	clusterCfg.IntVar(&maxPrio, "max-priority", 10, "maximum priority before a node should be considered not-ready.")
 	clusterCfg.IntVar(&minAvailableShards, "min-available-shards", 0, "minimum number of shards that must be available for a query to be handled.")
+	clusterCfg.IntVar(&gcPercentNotReady, "gc-percent-not-ready", gcPercent, "GOGC value to use when node is not ready.  Defaults to GOGC")
 	globalconf.Register("cluster", clusterCfg, flag.ExitOnError)
 
 	swimCfg := flag.NewFlagSet("swim", flag.ExitOnError)

cluster/node.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"runtime/debug"
 	"time"
 
 	"github.com/grafana/metrictank/tracing"
@@ -127,6 +128,18 @@ func (n HTTPNode) IsLocal() bool {
 	return n.local
 }
 
+// readyStateGCHandler adjusts the gcPercent value based on the node ready state
+func (n HTTPNode) readyStateGCHandler() {
+	if gcPercent == gcPercentNotReady {
+		return
+	}
+	if n.IsReady() {
+		debug.SetGCPercent(gcPercent)
+	} else {
+		debug.SetGCPercent(gcPercentNotReady)
+	}
+}
+
 // SetState sets the state of the node and returns whether the state changed
 func (n *HTTPNode) SetState(state NodeState) bool {
 	if n.State == state {
@@ -136,6 +149,7 @@ func (n *HTTPNode) SetState(state NodeState) bool {
 	now := time.Now()
 	n.Updated = now
 	n.StateChange = now
+	n.readyStateGCHandler()
 	return true
 }
 
@@ -146,6 +160,7 @@ func (n *HTTPNode) SetPriority(prio int) bool {
 	}
 	n.Priority = prio
 	n.Updated = time.Now()
+	n.readyStateGCHandler()
 	return true
 }
 

docker/docker-chaos/metrictank.ini

@@ -266,6 +266,9 @@ mode = single
 min-available-shards = 0
 # How long to wait before aborting http requests to cluster peers and returning a http 503 service unavailable
 http-timeout = 60s
+# GOGC value to use when node is not ready.  Defaults to GOGC
+# you can use this to set a more aggressive, latency-inducing GC behavior when the node is initializing and extra memory hungry
+# gc-percent-not-ready = 100
 
 ## SWIM clustering settings ##
 # only relevant when using cluster mode 'multi'

docker/docker-cluster/metrictank.ini

@@ -266,6 +266,9 @@ mode = single
 min-available-shards = 0
 # How long to wait before aborting http requests to cluster peers and returning a http 503 service unavailable
 http-timeout = 60s
+# GOGC value to use when node is not ready.  Defaults to GOGC
+# you can use this to set a more aggressive, latency-inducing GC behavior when the node is initializing and extra memory hungry
+# gc-percent-not-ready = 100
 
 ## SWIM clustering settings ##
 # only relevant when using cluster mode 'multi'

docker/docker-dev-custom-cfg-kafka/metrictank.ini

@@ -266,6 +266,9 @@ mode = single
 min-available-shards = 0
 # How long to wait before aborting http requests to cluster peers and returning a http 503 service unavailable
 http-timeout = 60s
+# GOGC value to use when node is not ready.  Defaults to GOGC
+# you can use this to set a more aggressive, latency-inducing GC behavior when the node is initializing and extra memory hungry
+# gc-percent-not-ready = 100
 
 ## SWIM clustering settings ##
 # only relevant when using cluster mode 'multi'

docs/config.md

@@ -322,6 +322,9 @@ mode = single
 min-available-shards = 0
 # How long to wait before aborting http requests to cluster peers and returning a http 503 service unavailable
 http-timeout = 60s
+# GOGC value to use when node is not ready.  Defaults to GOGC
+# you can use this to set a more aggressive, latency-inducing GC behavior when the node is initializing and extra memory hungry
+# gc-percent-not-ready = 100
 ```
 
 ## SWIM clustering settings ##

metrictank-sample.ini

@@ -269,6 +269,9 @@ mode = single
 min-available-shards = 0
 # How long to wait before aborting http requests to cluster peers and returning a http 503 service unavailable
 http-timeout = 60s
+# GOGC value to use when node is not ready.  Defaults to GOGC
+# you can use this to set a more aggressive, latency-inducing GC behavior when the node is initializing and extra memory hungry
+# gc-percent-not-ready = 100
 
 ## SWIM clustering settings ##
 # only relevant when using cluster mode 'multi'

scripts/config/metrictank-docker.ini

@@ -266,6 +266,9 @@ mode = single
 min-available-shards = 0
 # How long to wait before aborting http requests to cluster peers and returning a http 503 service unavailable
 http-timeout = 60s
+# GOGC value to use when node is not ready.  Defaults to GOGC
+# you can use this to set a more aggressive, latency-inducing GC behavior when the node is initializing and extra memory hungry
+# gc-percent-not-ready = 100
 
 ## SWIM clustering settings ##
 # only relevant when using cluster mode 'multi'

scripts/config/metrictank-package.ini

@@ -266,6 +266,9 @@ mode = single
 min-available-shards = 0
 # How long to wait before aborting http requests to cluster peers and returning a http 503 service unavailable
 http-timeout = 60s
+# GOGC value to use when node is not ready.  Defaults to GOGC
+# you can use this to set a more aggressive, latency-inducing GC behavior when the node is initializing and extra memory hungry
+# gc-percent-not-ready = 100
 
 ## SWIM clustering settings ##
 # only relevant when using cluster mode 'multi'