make findCache settings configurable

- add config settings for:
 - find-cache-invalidate-queue:  number of new series names to process
    Each new series name is compared against expresions in the findCache
    if the expression matches the new name, it is removed from the cache.
 - find-cache-backoff: amount of time to disable the findCache for when
   the invalidate-queue fills up.  If there are lots of new series being
   added then scanning the cache for patterns that match can become
   exspensive, so it is just best to disable the cache for a while.
- log a message when the cache is disabled due to the invalidateQueue
  limit being reached.

docker/docker-chaos/metrictank.ini

@@ -398,6 +398,10 @@ max-prune-lock-time = 100ms
 partitioned = false
 # number of find expressions to cache
 find-cache-size = 1000
+# size of queue for invalidating findCache entries.
+find-cache-invalidate-queue = 100
+# amount of time to disable the findCache when the invalidate queue fills up
+find-cache-backoff = 60s
 
 ### Bigtable index
 [bigtable-idx]

docker/docker-cluster/metrictank.ini

@@ -398,6 +398,10 @@ max-prune-lock-time = 100ms
 partitioned = false
 # number of find expressions to cache
 find-cache-size = 1000
+# size of queue for invalidating findCache entries.
+find-cache-invalidate-queue = 100
+# amount of time to disable the findCache when the invalidate queue fills up
+find-cache-backoff = 60s
 
 ### Bigtable index
 [bigtable-idx]

docker/docker-dev-custom-cfg-kafka/metrictank.ini

@@ -398,6 +398,10 @@ max-prune-lock-time = 100ms
 partitioned = false
 # number of find expressions to cache
 find-cache-size = 1000
+# size of queue for invalidating findCache entries.
+find-cache-invalidate-queue = 100
+# amount of time to disable the findCache when the invalidate queue fills up
+find-cache-backoff = 60s
 
 ### Bigtable index
 [bigtable-idx]

docs/config.md

@@ -465,6 +465,10 @@ max-prune-lock-time = 100ms
 partitioned = false
 # number of find expressions to cache
 find-cache-size = 1000
+# size of queue for invalidating findCache entries.
+find-cache-invalidate-queue = 100
+# amount of time to disable the findCache when the invalidate queue fills up
+find-cache-backoff = 60s
 ```
 
 ### Bigtable index

idx/memory/find_cache.go

@@ -16,6 +16,10 @@ var (
 	findCacheHit = stats.NewCounterRate32("idx.memory.find-cache.hit")
 	// metric idx.memory.find-cache.hit is a counter of findCache misses
 	findCacheMiss = stats.NewCounterRate32("idx.memory.find-cache.miss")
+
+	findCacheSize            = 1000
+	findCacheInvalidateQueue = 100
+	findCacheBackoff         = time.Minute
 )
 
 type FindCache struct {
@@ -70,7 +74,7 @@ func (c *FindCache) Add(orgId uint32, pattern string, nodes []*Node) {
 		}
 		c.Lock()
 		c.cache[orgId] = cache
-		c.newSeries[orgId] = make(chan struct{}, 100)
+		c.newSeries[orgId] = make(chan struct{}, findCacheInvalidateQueue)
 		c.Unlock()
 	}
 	cache.Add(pattern, nodes)
@@ -98,7 +102,7 @@ func (c *FindCache) InvalidateFor(orgId uint32, path string) {
 	case c.newSeries[orgId] <- struct{}{}:
 	default:
 		c.Lock()
-		c.backoff[orgId] = time.Now().Add(time.Minute)
+		c.backoff[orgId] = time.Now().Add(findCacheBackoff)
 		delete(c.cache, orgId)
 		c.Unlock()
 		for i := 0; i < len(c.newSeries[orgId]); i++ {
@@ -107,7 +111,7 @@ func (c *FindCache) InvalidateFor(orgId uint32, path string) {
 			default:
 			}
 		}
-
+		log.Infof("memory-idx: findCache invalidate-queue full. Disabling cache for %s. num-cached-entries=%d", findCacheBackoff.String(), cache.Len())
 		return
 	}
 

idx/memory/memory.go

@@ -56,7 +56,6 @@ var (
 	indexRulesFile      string
 	IndexRules          conf.IndexRules
 	Partitioned         bool
-	findCacheSize       = 1000
 )
 
 func ConfigSetup() {
@@ -67,6 +66,8 @@ func ConfigSetup() {
 	memoryIdx.IntVar(&TagQueryWorkers, "tag-query-workers", 50, "number of workers to spin up to evaluate tag queries")
 	memoryIdx.IntVar(&matchCacheSize, "match-cache-size", 1000, "size of regular expression cache in tag query evaluation")
 	memoryIdx.IntVar(&findCacheSize, "find-cache-size", 1000, "number of find expressions to cache")
+	memoryIdx.IntVar(&findCacheInvalidateQueue, "find-cache-invalidate-queue", 100, "size of queue for invalidating findCache entries.")
+	memoryIdx.DurationVar(&findCacheBackoff, "find-cache-backoff", time.Minute, "amount of time to disable the findCache when the invalidate queue fills up.")
 	memoryIdx.StringVar(&indexRulesFile, "rules-file", "/etc/metrictank/index-rules.conf", "path to index-rules.conf file")
 	memoryIdx.StringVar(&maxPruneLockTimeStr, "max-prune-lock-time", "100ms", "Maximum duration each second a prune job can lock the index.")
 	globalconf.Register("memory-idx", memoryIdx, flag.ExitOnError)

metrictank-sample.ini

@@ -401,6 +401,10 @@ max-prune-lock-time = 100ms
 partitioned = false
 # number of find expressions to cache
 find-cache-size = 1000
+# size of queue for invalidating findCache entries.
+find-cache-invalidate-queue = 100
+# amount of time to disable the findCache when the invalidate queue fills up
+find-cache-backoff = 60s
 
 ### Bigtable index
 [bigtable-idx]

scripts/config/metrictank-package.ini

@@ -398,6 +398,10 @@ max-prune-lock-time = 100ms
 partitioned = false
 # number of find expressions to cache
 find-cache-size = 1000
+# size of queue for invalidating findCache entries.
+find-cache-invalidate-queue = 100
+# amount of time to disable the findCache when the invalidate queue fills up
+find-cache-backoff = 60s
 
 ### Bigtable index
 [bigtable-idx]