Merge pull request #1175 from grafana/fix-1168

aggregated chunk GC fix + lower default metric-max-stale

docker/docker-chaos/metrictank.ini

@@ -12,7 +12,7 @@ drop-first-chunk = false
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from in-memory ring buffer.
-metric-max-stale = 6h
+metric-max-stale = 3h
 # Interval to run garbage collection job
 gc-interval = 1h
 

docker/docker-cluster/metrictank.ini

@@ -12,7 +12,7 @@ drop-first-chunk = false
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from in-memory ring buffer.
-metric-max-stale = 6h
+metric-max-stale = 3h
 # Interval to run garbage collection job
 gc-interval = 1h
 

docker/docker-dev-custom-cfg-kafka/metrictank.ini

@@ -12,7 +12,7 @@ drop-first-chunk = false
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from in-memory ring buffer.
-metric-max-stale = 6h
+metric-max-stale = 3h
 # Interval to run garbage collection job
 gc-interval = 1h
 

docs/config.md

@@ -43,7 +43,7 @@ drop-first-chunk = false
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from in-memory ring buffer.
-metric-max-stale = 6h
+metric-max-stale = 3h
 # Interval to run garbage collection job
 gc-interval = 1h
 # duration before secondary nodes start serving requests

mdata/aggmetric.go

@@ -604,24 +604,19 @@ func (a *AggMetric) GC(now, chunkMinTs, metricMinTs uint32) bool {
 		return false
 	}
 
-	if currentChunk.Series.Finished {
-		// already closed and should be saved, though we cant guarantee that.
-		// Check if we should just delete the metric from memory.
-		if a.lastWrite < metricMinTs {
-			return a.gcAggregators(now, chunkMinTs, metricMinTs)
-		}
-	} else {
-		// chunk hasn't been written to in a while, and is not yet closed. Let's close it and persist it if
-		// we are a primary
+	if !currentChunk.Series.Finished {
+		// chunk hasn't been written to in a while, and is not yet closed.
+		// Let's close it and persist it if we are a primary
 		log.Debugf("AM: Found stale Chunk, adding end-of-stream bytes. key: %v T0: %d", a.Key, currentChunk.Series.T0)
 		currentChunk.Finish()
+		a.pushToCache(currentChunk)
 		if cluster.Manager.IsPrimary() {
 			log.Debugf("AM: persist(): node is primary, saving chunk. %v T0: %d", a.Key, currentChunk.Series.T0)
 			// persist the chunk. If the writeQueue is full, then this will block.
 			a.persist(a.CurrentChunkPos)
 		}
 	}
-	return false
+	return a.gcAggregators(now, chunkMinTs, metricMinTs) && a.lastWrite < metricMinTs
 }
 
 // gcAggregators returns whether all aggregators are stale and can be removed

metrictank-sample.ini

@@ -15,7 +15,7 @@ drop-first-chunk = false
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from in-memory ring buffer.
-metric-max-stale = 6h
+metric-max-stale = 3h
 # Interval to run garbage collection job
 gc-interval = 1h
 

scripts/config/metrictank-docker.ini

@@ -12,7 +12,7 @@ drop-first-chunk = false
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from in-memory ring buffer.
-metric-max-stale = 6h
+metric-max-stale = 3h
 # Interval to run garbage collection job
 gc-interval = 1h
 

scripts/config/metrictank-package.ini

@@ -12,7 +12,7 @@ drop-first-chunk = false
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from in-memory ring buffer.
-metric-max-stale = 6h
+metric-max-stale = 3h
 # Interval to run garbage collection job
 gc-interval = 1h