From 6a129ee7964312a401d1916454f2816e213b994f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 16 Feb 2019 01:43:14 +0200 Subject: [PATCH 01/16] store/cache: switch to a 2Q algorithm --- pkg/store/cache.go | 50 +++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/pkg/store/cache.go b/pkg/store/cache.go index 5acd2c104c..4ba1f349db 100644 --- a/pkg/store/cache.go +++ b/pkg/store/cache.go @@ -3,7 +3,7 @@ package store import ( "sync" - lru "github.com/hashicorp/golang-lru/simplelru" + lru "github.com/hashicorp/golang-lru" "github.com/oklog/ulid" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/tsdb/labels" @@ -34,7 +34,7 @@ type cacheKeySeries uint64 type indexCache struct { mtx sync.Mutex - lru *lru.LRU + lru *lru.TwoQueueCache maxSize uint64 curSize uint64 @@ -44,6 +44,7 @@ type indexCache struct { current *prometheus.GaugeVec currentSize *prometheus.GaugeVec overflow *prometheus.CounterVec + evicted *prometheus.CounterVec } // newIndexCache creates a new LRU cache for index entries and ensures the total cache @@ -52,7 +53,7 @@ func newIndexCache(reg prometheus.Registerer, maxBytes uint64) (*indexCache, err c := &indexCache{ maxSize: maxBytes, } - evicted := prometheus.NewCounterVec(prometheus.CounterOpts{ + c.evicted = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "thanos_store_index_cache_items_evicted_total", Help: "Total number of items that were evicted from the index cache.", }, []string{"item_type"}) @@ -88,22 +89,10 @@ func newIndexCache(reg prometheus.Registerer, maxBytes uint64) (*indexCache, err }, []string{"item_type"}) // Initialize eviction metric with 0. - evicted.WithLabelValues(cacheTypePostings) - evicted.WithLabelValues(cacheTypeSeries) + c.evicted.WithLabelValues(cacheTypePostings) + c.evicted.WithLabelValues(cacheTypeSeries) - // Initialize LRU cache with a high size limit since we will manage evictions ourselves - // based on stored size. - onEvict := func(key, val interface{}) { - k := key.(cacheItem).keyType() - v := val.([]byte) - - evicted.WithLabelValues(k).Inc() - c.current.WithLabelValues(k).Dec() - c.currentSize.WithLabelValues(k).Sub(float64(len(v))) - - c.curSize -= uint64(len(v)) - } - l, err := lru.NewLRU(1e12, onEvict) + l, err := lru.New2Q(1e12) if err != nil { return nil, err } @@ -116,11 +105,32 @@ func newIndexCache(reg prometheus.Registerer, maxBytes uint64) (*indexCache, err }, func() float64 { return float64(maxBytes) })) - reg.MustRegister(c.requests, c.hits, c.added, evicted, c.current, c.currentSize) + reg.MustRegister(c.requests, c.hits, c.added, c.evicted, c.current, c.currentSize) } return c, nil } +// removeOldest removes the oldest key from the 2Q cache. +// First it removes the most recently added, and then it prioritises +// the most frequently used. +func (c *indexCache) removeOldest() { + keys := c.lru.Keys() + if len(keys) > 0 { + lastKey := keys[len(keys)-1] + val, _ := c.lru.Get(lastKey) + k := lastKey.(cacheItem).keyType() + v := val.([]byte) + + c.lru.Remove(lastKey) + + c.evicted.WithLabelValues(k).Inc() + c.current.WithLabelValues(k).Dec() + c.currentSize.WithLabelValues(k).Sub(float64(len(v))) + + c.curSize -= uint64(len(v)) + } +} + // ensureFits tries to make sure that the passed slice will fit into the LRU cache. // Returns true if it will fit. func (c *indexCache) ensureFits(b []byte) bool { @@ -128,7 +138,7 @@ func (c *indexCache) ensureFits(b []byte) bool { return false } for c.curSize+uint64(len(b)) > c.maxSize { - c.lru.RemoveOldest() + c.removeOldest() } return true } From 5de20e04ccc3696b1e2ebe017ce0c64595882e2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Tue, 30 Jul 2019 21:43:40 +0300 Subject: [PATCH 02/16] store: cache: factor out the implementation --- go.mod | 1 + go.sum | 13 +++++++++ pkg/store/cache/cache.go | 18 ++++++------- pkg/store/cache/storage.go | 55 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 pkg/store/cache/storage.go diff --git a/go.mod b/go.mod index 9873a47487..1d37f1e7b8 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/NYTimes/gziphandler v1.1.1 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da github.com/cespare/xxhash v1.1.0 + github.com/dgraph-io/ristretto v0.0.0-20190730015433-12477759384c github.com/fatih/structtag v1.0.0 github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 diff --git a/go.sum b/go.sum index 24b25203f3..1852233f3b 100644 --- a/go.sum +++ b/go.sum @@ -20,12 +20,15 @@ github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdII github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VictoriaMetrics/fastcache v1.5.1/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845/go.mod h1:c8Mh99Cw82nrsAnPgxQSZHkswVOJF7/MqZb1ZdvriLM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= @@ -45,6 +48,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4= @@ -52,12 +56,17 @@ github.com/cockroachdb/cockroach v0.0.0-20170608034007-84bc9597164f/go.mod h1:xe github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI= github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.0-20190730015433-12477759384c h1:tHNhdWy6i49JuTSSiqD2DfcjUWt/ApUV7MW45RTgVbI= +github.com/dgraph-io/ristretto v0.0.0-20190730015433-12477759384c/go.mod h1:UvZmzj8odp3S1nli6yEb1vLME8iJFBrRcw8rAJEiu9Q= github.com/dgrijalva/jwt-go v0.0.0-20161101193935-9ed569b5d1ac h1:xrQJVwQCGqDvOO7/0+RyIq5J2M3Q4ZF7Ug/BMQtML1E= github.com/dgrijalva/jwt-go v0.0.0-20161101193935-9ed569b5d1ac/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -90,6 +99,7 @@ github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNI github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goburrow/cache v0.1.0/go.mod h1:8oxkfud4hvjO4tNjEKZfEd+LrpDVDlBIauGYsWGEzio= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= @@ -97,6 +107,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= @@ -313,6 +324,8 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9 h1:5Cp3cVwpQP4aCQ6jx6dNLP3IarbYiuStmIzYu+BjQwY= +github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index 0e90443971..5adf09db34 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -1,12 +1,10 @@ package storecache import ( - "math" "sync" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - lru "github.com/hashicorp/golang-lru/simplelru" "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -53,7 +51,7 @@ type IndexCache struct { mtx sync.Mutex logger log.Logger - lru *lru.LRU + storage StorageCache maxSizeBytes uint64 maxItemSizeBytes uint64 @@ -163,11 +161,11 @@ func NewIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*In // Initialize LRU cache with a high size limit since we will manage evictions ourselves // based on stored size using `RemoveOldest` method. - l, err := lru.NewLRU(math.MaxInt64, c.onEvict) + storage, err := NewSimpleLRU(c.onEvict) if err != nil { return nil, err } - c.lru = l + c.storage = storage level.Info(logger).Log( "msg", "created index cache", @@ -196,7 +194,7 @@ func (c *IndexCache) get(typ string, key cacheKey) ([]byte, bool) { c.mtx.Lock() defer c.mtx.Unlock() - v, ok := c.lru.Get(key) + v, ok := c.storage.Get(key) if !ok { return nil, false } @@ -210,7 +208,7 @@ func (c *IndexCache) set(typ string, key cacheKey, val []byte) { c.mtx.Lock() defer c.mtx.Unlock() - if _, ok := c.lru.Get(key); ok { + if _, ok := c.storage.Get(key); ok { return } @@ -223,7 +221,7 @@ func (c *IndexCache) set(typ string, key cacheKey, val []byte) { // to ensure we don't waste huge amounts of space for something small. v := make([]byte, len(val)) copy(v, val) - c.lru.Add(key, v) + c.storage.Add(key, v) c.added.WithLabelValues(typ).Inc() c.currentSize.WithLabelValues(typ).Add(float64(size)) @@ -248,7 +246,7 @@ func (c *IndexCache) ensureFits(size uint64, typ string) bool { } for c.curSize+size > c.maxSizeBytes { - if _, _, ok := c.lru.RemoveOldest(); !ok { + if _, _, ok := c.storage.RemoveOldest(); !ok { level.Error(c.logger).Log( "msg", "LRU has nothing more to evict, but we still cannot allocate the item. Resetting cache.", "maxItemSizeBytes", c.maxItemSizeBytes, @@ -264,7 +262,7 @@ func (c *IndexCache) ensureFits(size uint64, typ string) bool { } func (c *IndexCache) reset() { - c.lru.Purge() + c.storage.Purge() c.current.Reset() c.currentSize.Reset() c.totalCurrentSize.Reset() diff --git a/pkg/store/cache/storage.go b/pkg/store/cache/storage.go new file mode 100644 index 0000000000..4cf8ad7b25 --- /dev/null +++ b/pkg/store/cache/storage.go @@ -0,0 +1,55 @@ +package storecache + +import ( + "github.com/dgraph-io/ristretto" + lru "github.com/hashicorp/golang-lru/simplelru" + "math" +) + +var _ = ristretto.Cache{} + +// StorageCache is a wrapper around typical Get()/Set() operations +// of a cache. Some might be a no-op on certain implementations. +type StorageCache interface { + Get(key interface{}) (val interface{}, ok bool) + Add(key interface{}, val interface{}) + RemoveOldest() (key interface{}, val interface{}, ok bool) + Purge() +} + +// SimpleLRU is a wrapper around a simple LRU data structure. +type SimpleLRU struct { + l *lru.LRU +} + +// Add adds the key with the specified value. +func (s *SimpleLRU) Add(key, val interface{}) { + s.l.Add(key, val) +} + +// Get gets the key's value. +func (s *SimpleLRU) Get(key interface{}) (interface{}, bool) { + return s.l.Get(key) +} + +// RemoveOldest removes the oldest key. +func (s *SimpleLRU) RemoveOldest() (interface{}, interface{}, bool) { + return s.l.RemoveOldest() +} + +// Purge purges the LRU. +func (s *SimpleLRU) Purge() { + s.l.Purge() +} + +// NewSimpleLRU returns a new simple LRU based cache storage which +// calls the given onEvict on eviction. +func NewSimpleLRU(onEvict func(key, val interface{})) (StorageCache, error) { + // Initialize LRU cache with a high size limit since we will manage evictions ourselves + // based on stored size using `RemoveOldest` method. + l, err := lru.NewLRU(math.MaxInt64, onEvict) + if err != nil { + return nil, err + } + return StorageCache(&SimpleLRU{l: l}), nil +} From 2899f5e85fe32240faa9290bdfc21280c052c629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Fri, 13 Sep 2019 19:53:41 +0300 Subject: [PATCH 03/16] store: add ristretto, fix build --- go.mod | 1 + go.sum | 11 +++++++ pkg/store/cache/cache_test.go | 8 ++--- pkg/store/cache/ristretto.go | 56 +++++++++++++++++++++++++++++++++++ pkg/store/cache/storage.go | 3 -- 5 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 pkg/store/cache/ristretto.go diff --git a/go.mod b/go.mod index 5853cf2619..d1e5d5759c 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect + github.com/dgraph-io/ristretto v0.0.0-20190911103809-abe3f183a9fc github.com/fatih/structtag v1.0.0 github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 diff --git a/go.sum b/go.sum index cd1a85f28d..9f7e02ca68 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,7 @@ github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdII github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/VictoriaMetrics/fastcache v1.5.1/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845/go.mod h1:c8Mh99Cw82nrsAnPgxQSZHkswVOJF7/MqZb1ZdvriLM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -35,6 +36,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZq github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -60,16 +63,22 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v0.0.0-20181017004759-096ff4a8a059/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.0-20190911103809-abe3f183a9fc h1:KvE5JBYmbIlNZ2CIaghMmt1VicUvat42zhUVvMsfo0Q= +github.com/dgraph-io/ristretto v0.0.0-20190911103809-abe3f183a9fc/go.mod h1:UvZmzj8odp3S1nli6yEb1vLME8iJFBrRcw8rAJEiu9Q= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda h1:NyywMz59neOoVRFDz+ccfKWxn784fiHMDnZSy6T+JXY= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -149,6 +158,7 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+ github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goburrow/cache v0.1.0/go.mod h1:8oxkfud4hvjO4tNjEKZfEd+LrpDVDlBIauGYsWGEzio= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= @@ -427,6 +437,7 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= diff --git a/pkg/store/cache/cache_test.go b/pkg/store/cache/cache_test.go index eb87d028e5..1839511b30 100644 --- a/pkg/store/cache/cache_test.go +++ b/pkg/store/cache/cache_test.go @@ -35,7 +35,7 @@ func TestIndexCache_AvoidsDeadlock(t *testing.T) { cache.curSize = size }) testutil.Ok(t, err) - cache.lru = l + cache.storage = StorageCache(&SimpleLRU{l: l}) cache.SetPostings(ulid.MustNew(0, nil), labels.Label{Name: "test2", Value: "1"}, []byte{42, 33, 14, 67, 11}) @@ -157,7 +157,7 @@ func TestIndexCache_MaxNumberOfItemsHit(t *testing.T) { l, err := simplelru.NewLRU(2, cache.onEvict) testutil.Ok(t, err) - cache.lru = l + cache.storage = StorageCache(&SimpleLRU{l: l}) id := ulid.MustNew(0, nil) @@ -299,7 +299,7 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - _, _, ok = cache.lru.RemoveOldest() + _, _, ok = cache.storage.RemoveOldest() testutil.Assert(t, ok, "something to remove") testutil.Equals(t, uint64(0), cache.curSize) @@ -314,7 +314,7 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(2), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - _, _, ok = cache.lru.RemoveOldest() + _, _, ok = cache.storage.RemoveOldest() testutil.Assert(t, !ok, "nothing to remove") lbls3 := labels.Label{Name: "test", Value: "124"} diff --git a/pkg/store/cache/ristretto.go b/pkg/store/cache/ristretto.go new file mode 100644 index 0000000000..9f6dff09b1 --- /dev/null +++ b/pkg/store/cache/ristretto.go @@ -0,0 +1,56 @@ +package storecache + +import ( + "github.com/dgraph-io/ristretto" +) + +// TinyLFU is a wrapper around Ristretto (TinyLFU). +type TinyLFU struct { + l *ristretto.Cache +} + +// Add adds the key with the specified value. +func (t *TinyLFU) Add(key, val interface{}) { + v := val.([]byte) + t.l.Set(key, val, int64(len(v))) +} + +// Get gets the key's value. +func (t *TinyLFU) Get(key interface{}) (interface{}, bool) { + return t.l.Get(key) +} + +// RemoveOldest removes the oldest key. +func (t *TinyLFU) RemoveOldest() (interface{}, interface{}, bool) { + // NOOP + return nil, nil, false +} + +// Purge purges the LRU. +func (t *TinyLFU) Purge() { + // Recreate the whole cache. + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1 * 1024 * 1024 * 100 * 10, + MaxCost: 1 * 1024 * 1024 * 100, + BufferItems: 64, + }) + // TODO: handle properly. + if err != nil { + panic(err) + } + t.l = cache +} + +// NewTinyLFU returns a new TinyLFU based cache storage which +// calls the given onEvict on eviction. +func NewTinyLFU(onEvict func(key, val interface{})) (StorageCache, error) { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1 * 1024 * 1024 * 100 * 10, + MaxCost: 1 * 1024 * 1024 * 100, + BufferItems: 64, + }) + if err != nil { + return nil, err + } + return StorageCache(&TinyLFU{l: cache}), nil +} diff --git a/pkg/store/cache/storage.go b/pkg/store/cache/storage.go index 4cf8ad7b25..32f4e5ba72 100644 --- a/pkg/store/cache/storage.go +++ b/pkg/store/cache/storage.go @@ -1,13 +1,10 @@ package storecache import ( - "github.com/dgraph-io/ristretto" lru "github.com/hashicorp/golang-lru/simplelru" "math" ) -var _ = ristretto.Cache{} - // StorageCache is a wrapper around typical Get()/Set() operations // of a cache. Some might be a no-op on certain implementations. type StorageCache interface { From 7f8055d1972da739f06beaddb13adcf8b4b4d04c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Fri, 20 Sep 2019 17:39:26 +0300 Subject: [PATCH 04/16] store: cache: goimports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- pkg/store/cache/storage.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/store/cache/storage.go b/pkg/store/cache/storage.go index 32f4e5ba72..058fc8c706 100644 --- a/pkg/store/cache/storage.go +++ b/pkg/store/cache/storage.go @@ -1,8 +1,9 @@ package storecache import ( - lru "github.com/hashicorp/golang-lru/simplelru" "math" + + lru "github.com/hashicorp/golang-lru/simplelru" ) // StorageCache is a wrapper around typical Get()/Set() operations From a8d15704ac48899dbb8e56735bdac309c8330637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 21 Sep 2019 17:59:21 +0300 Subject: [PATCH 05/16] store: add flag for switching between algorithms --- CHANGELOG.md | 4 +++ cmd/thanos/store.go | 6 ++++ go.mod | 2 +- go.sum | 11 ++---- pkg/store/cache/cache.go | 64 +++++++++++++++++++++++++++++------ pkg/store/cache/cache_test.go | 33 ++++++++++++++++-- pkg/store/cache/ristretto.go | 21 ++++-------- pkg/store/cache/simplelru.go | 44 ++++++++++++++++++++++++ pkg/store/cache/storage.go | 43 ----------------------- 9 files changed, 148 insertions(+), 80 deletions(-) create mode 100644 pkg/store/cache/simplelru.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 655005269f..4c59764bda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased +### Added + +- [#849](https://github.com/thanos-io/thanos/pull/849) Thanos Store got a new experimental feature: you can switch between different algorithms used for the index cache storage! The new algorithm is based on a pretty modern paper and it performs much better under pressure, has a much better hit ratio, and so on. Please test it out with `--index-cache-algorithm` - it can either be `lru` or `tinylfu`. Please report the results on our issue tracker or in our Slack so that we would know! + ## v0.7.0 - 2019.09.02 Accepted into CNCF: diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index c3f7ccb871..d3ab8d34f5 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -31,6 +31,9 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). Default("250MB").Bytes() + indexCacheAlgorithm := cmd.Flag("index-cache-algorithm", "Algorithm to use for the index cache."). + Default("lru").Enum("lru", "tinylfu") + chunkPoolSize := cmd.Flag("chunk-pool-size", "Maximum size of concurrently allocatable bytes for chunks."). Default("2GB").Bytes() @@ -72,6 +75,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string *clientCA, *httpBindAddr, uint64(*indexCacheSize), + *indexCacheAlgorithm, uint64(*chunkPoolSize), uint64(*maxSampleCount), int(*maxConcurrent), @@ -101,6 +105,7 @@ func runStore( clientCA string, httpBindAddr string, indexCacheSizeBytes uint64, + indexCacheAlgorithm string, chunkPoolSizeBytes uint64, maxSampleCount uint64, maxConcurrent int, @@ -134,6 +139,7 @@ func runStore( indexCache, err := storecache.NewIndexCache(logger, reg, storecache.Opts{ MaxSizeBytes: indexCacheSizeBytes, MaxItemSizeBytes: maxItemSizeBytes, + Algorithm: storecache.CacheAlgorithm(indexCacheAlgorithm), }) if err != nil { return errors.Wrap(err, "create index cache") diff --git a/go.mod b/go.mod index d1e5d5759c..22887d02c6 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/dgraph-io/ristretto v0.0.0-20190911103809-abe3f183a9fc + github.com/dgraph-io/ristretto v0.0.0-20190920234558-024ad72a079a github.com/fatih/structtag v1.0.0 github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 diff --git a/go.sum b/go.sum index 9f7e02ca68..925069b829 100644 --- a/go.sum +++ b/go.sum @@ -26,7 +26,6 @@ github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdII github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/VictoriaMetrics/fastcache v1.5.1/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845/go.mod h1:c8Mh99Cw82nrsAnPgxQSZHkswVOJF7/MqZb1ZdvriLM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -36,8 +35,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZq github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -63,18 +60,16 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v0.0.0-20181017004759-096ff4a8a059/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.0-20190911103809-abe3f183a9fc h1:KvE5JBYmbIlNZ2CIaghMmt1VicUvat42zhUVvMsfo0Q= -github.com/dgraph-io/ristretto v0.0.0-20190911103809-abe3f183a9fc/go.mod h1:UvZmzj8odp3S1nli6yEb1vLME8iJFBrRcw8rAJEiu9Q= +github.com/dgraph-io/ristretto v0.0.0-20190920234558-024ad72a079a h1:50qBIMUJW2KQyKpMzw4lhC18Uwn0/N3pOuZTbIN+JyM= +github.com/dgraph-io/ristretto v0.0.0-20190920234558-024ad72a079a/go.mod h1:jg4yDfbNNmxP2Nq5Z7MbyQrXyl/syIRF2LnbbxqViho= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda h1:NyywMz59neOoVRFDz+ccfKWxn784fiHMDnZSy6T+JXY= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= @@ -158,7 +153,6 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+ github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/goburrow/cache v0.1.0/go.mod h1:8oxkfud4hvjO4tNjEKZfEd+LrpDVDlBIauGYsWGEzio= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= @@ -437,7 +431,6 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index 2f8cf85285..391e619f76 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -65,13 +65,28 @@ type IndexCache struct { currentSize *prometheus.GaugeVec totalCurrentSize *prometheus.GaugeVec overflow *prometheus.CounterVec + + // keyData is true if the cache retains the information about the key types. + keyData bool } +// CacheAlgorithm is the caching algorithm that is used by the index cache. +type CacheAlgorithm string + +const ( + // LRUCache is the LRU-based cache. + LRUCache CacheAlgorithm = "lru" + // TinyLFUCache is the TinyLFU-based cache. + TinyLFUCache CacheAlgorithm = "tinylfu" +) + type Opts struct { // MaxSizeBytes represents overall maximum number of bytes cache can contain. MaxSizeBytes uint64 // MaxItemSizeBytes represents maximum size of single item. MaxItemSizeBytes uint64 + // Cache algorithm that will be used. + Algorithm CacheAlgorithm } // NewIndexCache creates a new thread-safe LRU cache for index entries and ensures the total cache @@ -159,24 +174,41 @@ func NewIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*In reg.MustRegister(c.requests, c.hits, c.added, c.evicted, c.current, c.currentSize, c.totalCurrentSize, c.overflow) } - // Initialize LRU cache with a high size limit since we will manage evictions ourselves - // based on stored size using `RemoveOldest` method. - storage, err := NewSimpleLRU(c.onEvict) - if err != nil { - return nil, err + if opts.Algorithm == "" { + opts.Algorithm = "tinylfu" + } + + switch opts.Algorithm { + case LRUCache: + // Initialize the LRU cache with a high size limit since we will manage evictions ourselves + // based on stored size using `RemoveOldest` method. + storage, err := NewSimpleLRU(c.lruOnEvict) + if err != nil { + return nil, err + } + c.storage = storage + c.keyData = true + default: + case TinyLFUCache: + storage, err := NewTinyLFU(func(key uint64, val interface{}, cost int64) { + entrySize := sliceHeaderSize + cost + c.curSize -= uint64(entrySize) + }, int64(c.maxSizeBytes)) + if err != nil { + return nil, err + } + c.storage = storage } - c.storage = storage level.Info(logger).Log( "msg", "created index cache", "maxItemSizeBytes", c.maxItemSizeBytes, "maxSizeBytes", c.maxSizeBytes, - "maxItems", "math.MaxInt64", ) return c, nil } -func (c *IndexCache) onEvict(key, val interface{}) { +func (c *IndexCache) lruOnEvict(key, val interface{}) { k := key.(cacheKey).keyType() entrySize := sliceHeaderSize + uint64(len(val.([]byte))) @@ -222,15 +254,20 @@ func (c *IndexCache) set(typ string, key cacheKey, val []byte) { v := make([]byte, len(val)) copy(v, val) c.storage.Add(key, v) + c.curSize += size + + if !c.keyData { + return + } c.added.WithLabelValues(typ).Inc() c.currentSize.WithLabelValues(typ).Add(float64(size)) c.totalCurrentSize.WithLabelValues(typ).Add(float64(size + key.size())) c.current.WithLabelValues(typ).Inc() - c.curSize += size + } -// ensureFits tries to make sure that the passed slice will fit into the LRU cache. +// ensureFits tries to make sure that the passed slice will fit into the cache. // Returns true if it will fit. func (c *IndexCache) ensureFits(size uint64, typ string) bool { if size > c.maxItemSizeBytes { @@ -245,6 +282,11 @@ func (c *IndexCache) ensureFits(size uint64, typ string) bool { return false } + // TinyLFU already manages the capacity restrictions for us. + if _, ok := c.storage.(*TinyLFU); ok { + return true + } + for c.curSize+size > c.maxSizeBytes { if _, _, ok := c.storage.RemoveOldest(); !ok { level.Error(c.logger).Log( @@ -275,6 +317,7 @@ func (c *IndexCache) SetPostings(b ulid.ULID, l labels.Label, v []byte) { c.set(cacheTypePostings, cacheKey{b, cacheKeyPostings(l)}, v) } +// Postings gets the postings from the index cache as identified by the ulid and labels. func (c *IndexCache) Postings(b ulid.ULID, l labels.Label) ([]byte, bool) { return c.get(cacheTypePostings, cacheKey{b, cacheKeyPostings(l)}) } @@ -285,6 +328,7 @@ func (c *IndexCache) SetSeries(b ulid.ULID, id uint64, v []byte) { c.set(cacheTypeSeries, cacheKey{b, cacheKeySeries(id)}, v) } +// Series gets the series data from the index cache as identified by the ulid and labels. func (c *IndexCache) Series(b ulid.ULID, id uint64) ([]byte, bool) { return c.get(cacheTypeSeries, cacheKey{b, cacheKeySeries(id)}) } diff --git a/pkg/store/cache/cache_test.go b/pkg/store/cache/cache_test.go index 1839511b30..a97361b400 100644 --- a/pkg/store/cache/cache_test.go +++ b/pkg/store/cache/cache_test.go @@ -25,13 +25,14 @@ func TestIndexCache_AvoidsDeadlock(t *testing.T) { cache, err := NewIndexCache(log.NewNopLogger(), metrics, Opts{ MaxItemSizeBytes: sliceHeaderSize + 5, MaxSizeBytes: sliceHeaderSize + 5, + Algorithm: "lru", }) testutil.Ok(t, err) l, err := simplelru.NewLRU(math.MaxInt64, func(key, val interface{}) { // Hack LRU to simulate broken accounting: evictions do not reduce current size. size := cache.curSize - cache.onEvict(key, val) + cache.lruOnEvict(key, val) cache.curSize = size }) testutil.Ok(t, err) @@ -77,6 +78,7 @@ func TestIndexCache_UpdateItem(t *testing.T) { cache, err := NewIndexCache(log.NewSyncLogger(errorLogger), metrics, Opts{ MaxItemSizeBytes: maxSize, MaxSizeBytes: maxSize, + Algorithm: "lru", }) testutil.Ok(t, err) @@ -152,10 +154,11 @@ func TestIndexCache_MaxNumberOfItemsHit(t *testing.T) { cache, err := NewIndexCache(log.NewNopLogger(), metrics, Opts{ MaxItemSizeBytes: 2*sliceHeaderSize + 10, MaxSizeBytes: 2*sliceHeaderSize + 10, + Algorithm: "lru", }) testutil.Ok(t, err) - l, err := simplelru.NewLRU(2, cache.onEvict) + l, err := simplelru.NewLRU(2, cache.lruOnEvict) testutil.Ok(t, err) cache.storage = StorageCache(&SimpleLRU{l: l}) @@ -185,6 +188,7 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { cache, err := NewIndexCache(log.NewNopLogger(), metrics, Opts{ MaxItemSizeBytes: 2*sliceHeaderSize + 5, MaxSizeBytes: 2*sliceHeaderSize + 5, + Algorithm: "lru", }) testutil.Ok(t, err) @@ -365,3 +369,28 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(5), promtest.ToFloat64(cache.hits.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.hits.WithLabelValues(cacheTypeSeries))) } + +// TestIndexCache_TinyLFU_Smoke runs the smoke tests for TinyLFU. +func TestIndexCache_TinyLFU_Smoke(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + metrics := prometheus.NewRegistry() + cache, err := NewIndexCache(log.NewNopLogger(), metrics, Opts{ + MaxItemSizeBytes: 2*sliceHeaderSize + 5, + MaxSizeBytes: 2*sliceHeaderSize + 5, + Algorithm: "lru", + }) + testutil.Ok(t, err) + + id := ulid.MustNew(0, nil) + lbls := labels.Label{Name: "test", Value: "123"} + + _, ok := cache.Postings(id, lbls) + testutil.Assert(t, !ok, "no such key") + + cache.SetPostings(id, lbls, []byte{42}) + + val, ok := cache.Postings(id, lbls) + testutil.Assert(t, ok, "postings not found") + testutil.Assert(t, val[0] == 42 && len(val) == 1, "byte slice contains other content") +} diff --git a/pkg/store/cache/ristretto.go b/pkg/store/cache/ristretto.go index 9f6dff09b1..a344aa619e 100644 --- a/pkg/store/cache/ristretto.go +++ b/pkg/store/cache/ristretto.go @@ -22,32 +22,23 @@ func (t *TinyLFU) Get(key interface{}) (interface{}, bool) { // RemoveOldest removes the oldest key. func (t *TinyLFU) RemoveOldest() (interface{}, interface{}, bool) { - // NOOP + // NOOP since TinyLFU is size restricted itself. return nil, nil, false } // Purge purges the LRU. func (t *TinyLFU) Purge() { - // Recreate the whole cache. - cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 1 * 1024 * 1024 * 100 * 10, - MaxCost: 1 * 1024 * 1024 * 100, - BufferItems: 64, - }) - // TODO: handle properly. - if err != nil { - panic(err) - } - t.l = cache + // NOOP since TinyLFU is size restricted itself. } // NewTinyLFU returns a new TinyLFU based cache storage which // calls the given onEvict on eviction. -func NewTinyLFU(onEvict func(key, val interface{})) (StorageCache, error) { +func NewTinyLFU(onEvict func(key uint64, val interface{}, cost int64), maxSize int64) (StorageCache, error) { cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 1 * 1024 * 1024 * 100 * 10, - MaxCost: 1 * 1024 * 1024 * 100, + NumCounters: maxSize * 10, + MaxCost: maxSize, BufferItems: 64, + OnEvict: onEvict, }) if err != nil { return nil, err diff --git a/pkg/store/cache/simplelru.go b/pkg/store/cache/simplelru.go new file mode 100644 index 0000000000..818eb06abe --- /dev/null +++ b/pkg/store/cache/simplelru.go @@ -0,0 +1,44 @@ +package storecache + +import ( + "math" + + lru "github.com/hashicorp/golang-lru/simplelru" +) + +// SimpleLRU is a wrapper around a simple LRU data structure. +type SimpleLRU struct { + l *lru.LRU +} + +// Add adds the key with the specified value. +func (s *SimpleLRU) Add(key, val interface{}) { + s.l.Add(key, val) +} + +// Get gets the key's value. +func (s *SimpleLRU) Get(key interface{}) (interface{}, bool) { + return s.l.Get(key) +} + +// RemoveOldest removes the oldest key. +func (s *SimpleLRU) RemoveOldest() (interface{}, interface{}, bool) { + return s.l.RemoveOldest() +} + +// Purge purges the LRU. +func (s *SimpleLRU) Purge() { + s.l.Purge() +} + +// NewSimpleLRU returns a new simple LRU based cache storage which +// calls the given onEvict on eviction. +func NewSimpleLRU(onEvict func(key, val interface{})) (StorageCache, error) { + // Initialize LRU cache with a high size limit since we will manage evictions ourselves + // based on stored size using `RemoveOldest` method. + l, err := lru.NewLRU(math.MaxInt64, onEvict) + if err != nil { + return nil, err + } + return StorageCache(&SimpleLRU{l: l}), nil +} diff --git a/pkg/store/cache/storage.go b/pkg/store/cache/storage.go index 058fc8c706..c74a847c60 100644 --- a/pkg/store/cache/storage.go +++ b/pkg/store/cache/storage.go @@ -1,11 +1,5 @@ package storecache -import ( - "math" - - lru "github.com/hashicorp/golang-lru/simplelru" -) - // StorageCache is a wrapper around typical Get()/Set() operations // of a cache. Some might be a no-op on certain implementations. type StorageCache interface { @@ -14,40 +8,3 @@ type StorageCache interface { RemoveOldest() (key interface{}, val interface{}, ok bool) Purge() } - -// SimpleLRU is a wrapper around a simple LRU data structure. -type SimpleLRU struct { - l *lru.LRU -} - -// Add adds the key with the specified value. -func (s *SimpleLRU) Add(key, val interface{}) { - s.l.Add(key, val) -} - -// Get gets the key's value. -func (s *SimpleLRU) Get(key interface{}) (interface{}, bool) { - return s.l.Get(key) -} - -// RemoveOldest removes the oldest key. -func (s *SimpleLRU) RemoveOldest() (interface{}, interface{}, bool) { - return s.l.RemoveOldest() -} - -// Purge purges the LRU. -func (s *SimpleLRU) Purge() { - s.l.Purge() -} - -// NewSimpleLRU returns a new simple LRU based cache storage which -// calls the given onEvict on eviction. -func NewSimpleLRU(onEvict func(key, val interface{})) (StorageCache, error) { - // Initialize LRU cache with a high size limit since we will manage evictions ourselves - // based on stored size using `RemoveOldest` method. - l, err := lru.NewLRU(math.MaxInt64, onEvict) - if err != nil { - return nil, err - } - return StorageCache(&SimpleLRU{l: l}), nil -} From 47d97b9a4bbf24e03d0ff06e56491cfbb115b2a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 21 Sep 2019 18:01:47 +0300 Subject: [PATCH 06/16] docs/components/store: update --- docs/components/store.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/components/store.md b/docs/components/store.md index 8520a22a58..27615d4572 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -65,6 +65,8 @@ Flags: verification on server side. (tls.NoClientCert) --data-dir="./data" Data directory in which to cache remote blocks. --index-cache-size=250MB Maximum size of items held in the index cache. + --index-cache-algorithm=lru + Algorithm to use for the index cache. --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes for chunks. --store.grpc.series-sample-limit=0 From e1e4a8892b44774bbaa642b15be3aa5c054220da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 25 Sep 2019 15:42:50 +0300 Subject: [PATCH 07/16] cache: specify custom hashing function --- go.mod | 2 +- go.sum | 4 ++-- pkg/store/cache/ristretto.go | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 22887d02c6..b09f495f2b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/dgraph-io/ristretto v0.0.0-20190920234558-024ad72a079a + github.com/dgraph-io/ristretto v0.0.0-20190924012251-ae326c3348e8 github.com/fatih/structtag v1.0.0 github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 diff --git a/go.sum b/go.sum index 925069b829..6039be415d 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.0-20190920234558-024ad72a079a h1:50qBIMUJW2KQyKpMzw4lhC18Uwn0/N3pOuZTbIN+JyM= -github.com/dgraph-io/ristretto v0.0.0-20190920234558-024ad72a079a/go.mod h1:jg4yDfbNNmxP2Nq5Z7MbyQrXyl/syIRF2LnbbxqViho= +github.com/dgraph-io/ristretto v0.0.0-20190924012251-ae326c3348e8 h1:cg3kuXalA5TK0xWwanrvayAzN9Yg6xawalxESTGN7UM= +github.com/dgraph-io/ristretto v0.0.0-20190924012251-ae326c3348e8/go.mod h1:jg4yDfbNNmxP2Nq5Z7MbyQrXyl/syIRF2LnbbxqViho= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda h1:NyywMz59neOoVRFDz+ccfKWxn784fiHMDnZSy6T+JXY= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= diff --git a/pkg/store/cache/ristretto.go b/pkg/store/cache/ristretto.go index a344aa619e..0265860208 100644 --- a/pkg/store/cache/ristretto.go +++ b/pkg/store/cache/ristretto.go @@ -2,6 +2,7 @@ package storecache import ( "github.com/dgraph-io/ristretto" + "github.com/dgraph-io/ristretto/z" ) // TinyLFU is a wrapper around Ristretto (TinyLFU). @@ -39,6 +40,11 @@ func NewTinyLFU(onEvict func(key uint64, val interface{}, cost int64), maxSize i MaxCost: maxSize, BufferItems: 64, OnEvict: onEvict, + KeyToHash: func(key interface{}) uint64 { + k := key.(cacheKey) + s := z.KeyToHash(z.KeyToHash([16]byte(k.block)) + z.KeyToHash(k.key)) + return s + }, }) if err != nil { return nil, err From 4a42dc4a50d83f873112d5b387cc5d7d5b47af07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 25 Sep 2019 16:27:34 +0300 Subject: [PATCH 08/16] cache: refactor .keyData, set sane counters number --- pkg/store/cache/cache.go | 6 +----- pkg/store/cache/ristretto.go | 25 +++++++++++++++++++++---- pkg/store/cache/simplelru.go | 8 ++++++++ pkg/store/cache/storage.go | 1 + 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index 391e619f76..9e50df4821 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -65,9 +65,6 @@ type IndexCache struct { currentSize *prometheus.GaugeVec totalCurrentSize *prometheus.GaugeVec overflow *prometheus.CounterVec - - // keyData is true if the cache retains the information about the key types. - keyData bool } // CacheAlgorithm is the caching algorithm that is used by the index cache. @@ -187,7 +184,6 @@ func NewIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*In return nil, err } c.storage = storage - c.keyData = true default: case TinyLFUCache: storage, err := NewTinyLFU(func(key uint64, val interface{}, cost int64) { @@ -256,7 +252,7 @@ func (c *IndexCache) set(typ string, key cacheKey, val []byte) { c.storage.Add(key, v) c.curSize += size - if !c.keyData { + if !c.storage.KeyData() { return } diff --git a/pkg/store/cache/ristretto.go b/pkg/store/cache/ristretto.go index 0265860208..404ddbb028 100644 --- a/pkg/store/cache/ristretto.go +++ b/pkg/store/cache/ristretto.go @@ -32,18 +32,35 @@ func (t *TinyLFU) Purge() { // NOOP since TinyLFU is size restricted itself. } +// KeyData returns if the cache retains key data. +func (t *TinyLFU) KeyData() bool { + return true +} + // NewTinyLFU returns a new TinyLFU based cache storage which // calls the given onEvict on eviction. func NewTinyLFU(onEvict func(key uint64, val interface{}, cost int64), maxSize int64) (StorageCache, error) { cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: maxSize * 10, + NumCounters: 2 * 1000 * 1000, // TODO(GiedriusS): should this be configurable? MaxCost: maxSize, - BufferItems: 64, + BufferItems: 64, // Value that should give good enough performance. OnEvict: onEvict, KeyToHash: func(key interface{}) uint64 { k := key.(cacheKey) - s := z.KeyToHash(z.KeyToHash([16]byte(k.block)) + z.KeyToHash(k.key)) - return s + b := [16]byte(k.block) + + var d uint64 + + keyType := k.keyType() + switch keyType { + case cacheTypePostings: + datum := k.key.(cacheKeyPostings) + d = z.KeyToHash(datum.Name + datum.Value) + case cacheTypeSeries: + datum := k.key.(cacheKeySeries) + d = z.KeyToHash(uint64(datum)) + } + return z.KeyToHash(z.KeyToHash(b[:]) + d) }, }) if err != nil { diff --git a/pkg/store/cache/simplelru.go b/pkg/store/cache/simplelru.go index 818eb06abe..7ac1cd3ef2 100644 --- a/pkg/store/cache/simplelru.go +++ b/pkg/store/cache/simplelru.go @@ -9,6 +9,9 @@ import ( // SimpleLRU is a wrapper around a simple LRU data structure. type SimpleLRU struct { l *lru.LRU + + // keyData is true if the cache retains the information about the key types. + keyData bool } // Add adds the key with the specified value. @@ -31,6 +34,11 @@ func (s *SimpleLRU) Purge() { s.l.Purge() } +// KeyData returns if the cache retains key data. +func (s *SimpleLRU) KeyData() bool { + return true +} + // NewSimpleLRU returns a new simple LRU based cache storage which // calls the given onEvict on eviction. func NewSimpleLRU(onEvict func(key, val interface{})) (StorageCache, error) { diff --git a/pkg/store/cache/storage.go b/pkg/store/cache/storage.go index c74a847c60..adfbc4a5bb 100644 --- a/pkg/store/cache/storage.go +++ b/pkg/store/cache/storage.go @@ -7,4 +7,5 @@ type StorageCache interface { Add(key interface{}, val interface{}) RemoveOldest() (key interface{}, val interface{}, ok bool) Purge() + KeyData() bool // True if it retains exact information about keys. } From 36c0d18fdf8e6ba4f55af00e3177220c24cc3c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 25 Sep 2019 16:38:04 +0300 Subject: [PATCH 09/16] cache: ristretto: make ctrs dynamic --- pkg/store/cache/ristretto.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/store/cache/ristretto.go b/pkg/store/cache/ristretto.go index 404ddbb028..629a7ac9c5 100644 --- a/pkg/store/cache/ristretto.go +++ b/pkg/store/cache/ristretto.go @@ -40,8 +40,9 @@ func (t *TinyLFU) KeyData() bool { // NewTinyLFU returns a new TinyLFU based cache storage which // calls the given onEvict on eviction. func NewTinyLFU(onEvict func(key uint64, val interface{}, cost int64), maxSize int64) (StorageCache, error) { + ctrs := maxSize / 1000 // Seems like a good value, ad-hoc calculation of cache size divided by avg. cache item's size. cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 2 * 1000 * 1000, // TODO(GiedriusS): should this be configurable? + NumCounters: ctrs, MaxCost: maxSize, BufferItems: 64, // Value that should give good enough performance. OnEvict: onEvict, From b815c1b9fce0c586d65607d9b2141fab46362d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 25 Sep 2019 16:42:12 +0300 Subject: [PATCH 10/16] cache: simplelru: remove unused member --- pkg/store/cache/simplelru.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/store/cache/simplelru.go b/pkg/store/cache/simplelru.go index 7ac1cd3ef2..a425179899 100644 --- a/pkg/store/cache/simplelru.go +++ b/pkg/store/cache/simplelru.go @@ -9,9 +9,6 @@ import ( // SimpleLRU is a wrapper around a simple LRU data structure. type SimpleLRU struct { l *lru.LRU - - // keyData is true if the cache retains the information about the key types. - keyData bool } // Add adds the key with the specified value. From 6a157918d6c5537124e866f827ecdf2853a50c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 16 Nov 2019 00:15:02 +0200 Subject: [PATCH 11/16] *: bring ristretto up-to-date MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- cmd/thanos/store.go | 2 +- docs/components/store.md | 2 -- go.mod | 4 +++- go.sum | 4 ++-- pkg/store/cache/cache.go | 2 +- pkg/store/cache/ristretto.go | 11 ++++++----- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index d3ab8d34f5..42e8cb400d 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -32,7 +32,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string Default("250MB").Bytes() indexCacheAlgorithm := cmd.Flag("index-cache-algorithm", "Algorithm to use for the index cache."). - Default("lru").Enum("lru", "tinylfu") + Default("lru").Hidden().Enum("lru", "tinylfu") chunkPoolSize := cmd.Flag("chunk-pool-size", "Maximum size of concurrently allocatable bytes for chunks."). Default("2GB").Bytes() diff --git a/docs/components/store.md b/docs/components/store.md index 27615d4572..8520a22a58 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -65,8 +65,6 @@ Flags: verification on server side. (tls.NoClientCert) --data-dir="./data" Data directory in which to cache remote blocks. --index-cache-size=250MB Maximum size of items held in the index cache. - --index-cache-algorithm=lru - Algorithm to use for the index cache. --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes for chunks. --store.grpc.series-sample-limit=0 diff --git a/go.mod b/go.mod index b09f495f2b..ede77b37f6 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/dgraph-io/ristretto v0.0.0-20190924012251-ae326c3348e8 + github.com/dgraph-io/ristretto v0.0.0-20191114170855-99d1bbbf28e6 github.com/fatih/structtag v1.0.0 github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 @@ -67,3 +67,5 @@ replace ( k8s.io/klog => k8s.io/klog v0.3.1 k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 ) + +go 1.13 diff --git a/go.sum b/go.sum index 6039be415d..145b1352bb 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.0-20190924012251-ae326c3348e8 h1:cg3kuXalA5TK0xWwanrvayAzN9Yg6xawalxESTGN7UM= -github.com/dgraph-io/ristretto v0.0.0-20190924012251-ae326c3348e8/go.mod h1:jg4yDfbNNmxP2Nq5Z7MbyQrXyl/syIRF2LnbbxqViho= +github.com/dgraph-io/ristretto v0.0.0-20191114170855-99d1bbbf28e6 h1:liDEMz8LbPxfuI8e/noprwccn6gZGv2rN1AgucbxjHs= +github.com/dgraph-io/ristretto v0.0.0-20191114170855-99d1bbbf28e6/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda h1:NyywMz59neOoVRFDz+ccfKWxn784fiHMDnZSy6T+JXY= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index 9e50df4821..dc3d1712d5 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -186,7 +186,7 @@ func NewIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*In c.storage = storage default: case TinyLFUCache: - storage, err := NewTinyLFU(func(key uint64, val interface{}, cost int64) { + storage, err := NewTinyLFU(func(key uint64, conflict uint64, val interface{}, cost int64) { entrySize := sliceHeaderSize + cost c.curSize -= uint64(entrySize) }, int64(c.maxSizeBytes)) diff --git a/pkg/store/cache/ristretto.go b/pkg/store/cache/ristretto.go index 629a7ac9c5..2a5e24adcf 100644 --- a/pkg/store/cache/ristretto.go +++ b/pkg/store/cache/ristretto.go @@ -39,14 +39,14 @@ func (t *TinyLFU) KeyData() bool { // NewTinyLFU returns a new TinyLFU based cache storage which // calls the given onEvict on eviction. -func NewTinyLFU(onEvict func(key uint64, val interface{}, cost int64), maxSize int64) (StorageCache, error) { +func NewTinyLFU(onEvict func(key uint64, conflict uint64, val interface{}, cost int64), maxSize int64) (StorageCache, error) { ctrs := maxSize / 1000 // Seems like a good value, ad-hoc calculation of cache size divided by avg. cache item's size. cache, err := ristretto.NewCache(&ristretto.Config{ NumCounters: ctrs, MaxCost: maxSize, BufferItems: 64, // Value that should give good enough performance. OnEvict: onEvict, - KeyToHash: func(key interface{}) uint64 { + KeyToHash: func(key interface{}) (uint64, uint64) { k := key.(cacheKey) b := [16]byte(k.block) @@ -56,12 +56,13 @@ func NewTinyLFU(onEvict func(key uint64, val interface{}, cost int64), maxSize i switch keyType { case cacheTypePostings: datum := k.key.(cacheKeyPostings) - d = z.KeyToHash(datum.Name + datum.Value) + d, _ = z.KeyToHash(datum.Name + datum.Value) case cacheTypeSeries: datum := k.key.(cacheKeySeries) - d = z.KeyToHash(uint64(datum)) + d, _ = z.KeyToHash(uint64(datum)) } - return z.KeyToHash(z.KeyToHash(b[:]) + d) + hashedBlock, _ := z.KeyToHash(b[:]) + return z.KeyToHash(hashedBlock + d) }, }) if err != nil { From 354963561d7a9c14d2683ed1b6bbe98d5ecbdc2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 16 Nov 2019 00:33:50 +0200 Subject: [PATCH 12/16] store: cache: rework key store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encode the key's type inside of the value itself in the tinylfu case. This is needed so that we would not lose any information in the metrics about the different types of keys. 1 byte for this seems like a good enough trade-off for me. Signed-off-by: Giedrius Statkevičius --- pkg/store/cache/cache.go | 49 +++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index dc3d1712d5..9de66c9dcb 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -172,7 +172,7 @@ func NewIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*In } if opts.Algorithm == "" { - opts.Algorithm = "tinylfu" + opts.Algorithm = "lru" } switch opts.Algorithm { @@ -188,7 +188,25 @@ func NewIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*In case TinyLFUCache: storage, err := NewTinyLFU(func(key uint64, conflict uint64, val interface{}, cost int64) { entrySize := sliceHeaderSize + cost + + // Extract the key's type encoded as the last byte. + v := val.([]byte) + k := v[len(v)-1] + var keyType string + switch k { + case 0: + keyType = cacheTypeSeries + case 1: + keyType = cacheTypePostings + default: + panic("unhandled key type") + } + c.curSize -= uint64(entrySize) + c.evicted.WithLabelValues(keyType).Inc() + c.current.WithLabelValues(keyType).Dec() + c.currentSize.WithLabelValues(keyType).Sub(float64(entrySize)) + c.totalCurrentSize.WithLabelValues(keyType).Sub(float64(entrySize + 8)) }, int64(c.maxSizeBytes)) if err != nil { return nil, err @@ -245,20 +263,35 @@ func (c *IndexCache) set(typ string, key cacheKey, val []byte) { return } + var keySize uint64 // The caller may be passing in a sub-slice of a huge array. Copy the data // to ensure we don't waste huge amounts of space for something small. - v := make([]byte, len(val)) - copy(v, val) - c.storage.Add(key, v) - c.curSize += size - + var v []byte if !c.storage.KeyData() { - return + v = make([]byte, len(val)+1) + copy(v, val) + // Encode the key's type inside of the value. + switch typ { + case cacheTypeSeries: + v = append(v, byte(0)) + case cacheTypePostings: + v = append(v, byte(1)) + default: + panic("unhandled index cache item type") + } + size++ + keySize = 8 + } else { + v = make([]byte, len(val)) + copy(v, val) + keySize = key.size() } + c.storage.Add(key, v) + c.curSize += size c.added.WithLabelValues(typ).Inc() c.currentSize.WithLabelValues(typ).Add(float64(size)) - c.totalCurrentSize.WithLabelValues(typ).Add(float64(size + key.size())) + c.totalCurrentSize.WithLabelValues(typ).Add(float64(size + keySize)) c.current.WithLabelValues(typ).Inc() } From 23e4de041c612d9b10985e393758dd28e30ea05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 16 Nov 2019 00:36:42 +0200 Subject: [PATCH 13/16] store: cache: ristretto: fix missing methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an implementation for Purge() which calls Clear(). Fix KeyData() return value to return `false` which is the true value. Signed-off-by: Giedrius Statkevičius --- pkg/store/cache/ristretto.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/store/cache/ristretto.go b/pkg/store/cache/ristretto.go index 2a5e24adcf..1d3bef732a 100644 --- a/pkg/store/cache/ristretto.go +++ b/pkg/store/cache/ristretto.go @@ -29,12 +29,12 @@ func (t *TinyLFU) RemoveOldest() (interface{}, interface{}, bool) { // Purge purges the LRU. func (t *TinyLFU) Purge() { - // NOOP since TinyLFU is size restricted itself. + t.l.Clear() } // KeyData returns if the cache retains key data. func (t *TinyLFU) KeyData() bool { - return true + return false } // NewTinyLFU returns a new TinyLFU based cache storage which From 3e64693d82683f848bf2aeeb874ddcdeacbbe684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 16 Nov 2019 01:14:58 +0200 Subject: [PATCH 14/16] *: fix post-merge debris MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- cmd/thanos/store.go | 14 +++----------- go.sum | 5 +++-- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 2e18cb722c..eba84160a5 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -155,6 +155,8 @@ func runStore( return errors.Wrap(err, "create bucket client") } + // TODO(bwplotka): Add as a flag? + maxItemSizeBytes := indexCacheSizeBytes / 2 indexCache, err := storecache.NewIndexCache(logger, reg, storecache.Opts{ MaxSizeBytes: indexCacheSizeBytes, MaxItemSizeBytes: maxItemSizeBytes, @@ -163,6 +165,7 @@ func runStore( if err != nil { return errors.Wrap(err, "create index cache") } + relabelContentYaml, err := selectorRelabelConf.Content() if err != nil { return errors.Wrap(err, "get content of relabel configuration") @@ -180,17 +183,6 @@ func runStore( } }() - // TODO(bwplotka): Add as a flag? - maxItemSizeBytes := indexCacheSizeBytes / 2 - - indexCache, err := storecache.NewIndexCache(logger, reg, storecache.Opts{ - MaxSizeBytes: indexCacheSizeBytes, - MaxItemSizeBytes: maxItemSizeBytes, - }) - if err != nil { - return errors.Wrap(err, "create index cache") - } - bs, err := store.NewBucketStore( logger, reg, diff --git a/go.sum b/go.sum index 1f6ac6a45c..170eee0840 100644 --- a/go.sum +++ b/go.sum @@ -126,10 +126,11 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/ristretto v0.0.0-20191114170855-99d1bbbf28e6 h1:liDEMz8LbPxfuI8e/noprwccn6gZGv2rN1AgucbxjHs= github.com/dgraph-io/ristretto v0.0.0-20191114170855-99d1bbbf28e6/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= From 02d39f27bf331ddb8735b9e3d4a65d630612b58b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 16 Nov 2019 01:29:40 +0200 Subject: [PATCH 15/16] store: cache: refactor constants into variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- pkg/store/cache/cache.go | 8 ++++---- pkg/store/cache/ristretto.go | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index cce184f99a..d5e6126fc1 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -194,9 +194,9 @@ func NewIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*In k := v[len(v)-1] var keyType string switch k { - case 0: + case keyTypePostings: keyType = cacheTypeSeries - case 1: + case keyTypeSeries: keyType = cacheTypePostings default: panic("unhandled key type") @@ -273,9 +273,9 @@ func (c *IndexCache) set(typ string, key cacheKey, val []byte) { // Encode the key's type inside of the value. switch typ { case cacheTypeSeries: - v = append(v, byte(0)) + v = append(v, keyTypeSeries) case cacheTypePostings: - v = append(v, byte(1)) + v = append(v, keyTypePostings) default: panic("unhandled index cache item type") } diff --git a/pkg/store/cache/ristretto.go b/pkg/store/cache/ristretto.go index 1d3bef732a..79ce663ec1 100644 --- a/pkg/store/cache/ristretto.go +++ b/pkg/store/cache/ristretto.go @@ -5,6 +5,12 @@ import ( "github.com/dgraph-io/ristretto/z" ) +// Values used for encoding the key's type. +const ( + keyTypePostings byte = iota + keyTypeSeries +) + // TinyLFU is a wrapper around Ristretto (TinyLFU). type TinyLFU struct { l *ristretto.Cache From 95bce84e684275790bb1b99fca542a614f24da3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 16 Nov 2019 01:30:51 +0200 Subject: [PATCH 16/16] store: cache: fix size calculation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two uint64 hashes are used. Signed-off-by: Giedrius Statkevičius --- pkg/store/cache/cache.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index d5e6126fc1..47db6fcaf5 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -206,7 +206,8 @@ func NewIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*In c.evicted.WithLabelValues(keyType).Inc() c.current.WithLabelValues(keyType).Dec() c.currentSize.WithLabelValues(keyType).Sub(float64(entrySize)) - c.totalCurrentSize.WithLabelValues(keyType).Sub(float64(entrySize + 8)) + // uint64 keys are used and uint64 hashes for checking conflicts. + c.totalCurrentSize.WithLabelValues(keyType).Sub(float64(entrySize + 8 + 8)) }, int64(c.maxSizeBytes)) if err != nil { return nil, err @@ -280,7 +281,8 @@ func (c *IndexCache) set(typ string, key cacheKey, val []byte) { panic("unhandled index cache item type") } size++ - keySize = 8 + // 2 uint64 hashes. + keySize = 8 + 8 } else { v = make([]byte, len(val)) copy(v, val)