From ac5bb501328da7ff93af110f41f4b62562b29349 Mon Sep 17 00:00:00 2001 From: Vincent Boutour Date: Sat, 3 Sep 2022 12:47:36 +0200 Subject: [PATCH] refactor: Using cache layer more efficiently Signed-off-by: Vincent Boutour --- go.mod | 6 +++--- go.sum | 12 ++++++------ pkg/exif/aggregate.go | 26 ++++++-------------------- pkg/exif/event.go | 6 +++--- pkg/exif/exif.go | 22 +++++++++++++++------- pkg/thumbnail/thumbnail.go | 11 +++++++++-- pkg/thumbnail/utils.go | 5 +---- pkg/thumbnail/utils_test.go | 2 ++ 8 files changed, 45 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index 2e5717fb..4963e269 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/ViBiOh/fibr go 1.19 require ( - github.com/ViBiOh/ChatPotte v0.2.5 + github.com/ViBiOh/ChatPotte v0.2.6 github.com/ViBiOh/absto v1.0.3 - github.com/ViBiOh/auth/v2 v2.14.2 + github.com/ViBiOh/auth/v2 v2.14.3 github.com/ViBiOh/exas v0.5.0 github.com/ViBiOh/flags v1.2.0 - github.com/ViBiOh/httputils/v4 v4.47.0 + github.com/ViBiOh/httputils/v4 v4.47.1 github.com/ViBiOh/vith v0.5.0 github.com/golang/mock v1.6.0 github.com/prometheus/client_golang v1.13.0 diff --git a/go.sum b/go.sum index 6b588404..bc013683 100644 --- a/go.sum +++ b/go.sum @@ -33,18 +33,18 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ViBiOh/ChatPotte v0.2.5 h1:nZrC0RN5RT0DRGHlpVUhhOWUPFeLKq18P35I9ZMacuM= -github.com/ViBiOh/ChatPotte v0.2.5/go.mod h1:H/B3xIv0A3eddp61mlENHg7P0DEu3hJJnVdD6bRkzhs= +github.com/ViBiOh/ChatPotte v0.2.6 h1:7neA/C6vEPlnNW2ymlWQf2wmp8WUSQsJz/2iV18l18M= +github.com/ViBiOh/ChatPotte v0.2.6/go.mod h1:hGY0JXNxQOLa1Lc7G7j3xpN3PltI9MjpXoI3dWR8e5s= github.com/ViBiOh/absto v1.0.3 h1:guwVdRN7mIJM6XO65ABqDjz/7JiFMGJ7+VfmE/Ni3Zw= github.com/ViBiOh/absto v1.0.3/go.mod h1:nu8N6LQnyxYhdnTAOtlpr4zca2tOcj5uUgI30oCF4vo= -github.com/ViBiOh/auth/v2 v2.14.2 h1:3z89LtCGs8zZFLOTzHiN3YIacZ2UDxzpkQ/sO2XJcfY= -github.com/ViBiOh/auth/v2 v2.14.2/go.mod h1:wktrqt4O8toixNWekhJ6cL8cXXMOLCSlKBhWUIpHDto= +github.com/ViBiOh/auth/v2 v2.14.3 h1:PHkryZKMG/CRNAqgM7yxXzLI9GRYytxX8OaO7XVsk4E= +github.com/ViBiOh/auth/v2 v2.14.3/go.mod h1:+gm2mDOj/178egUVCqrug7tOwI32IkZsofT1YaL+jJY= github.com/ViBiOh/exas v0.5.0 h1:ZL5v5rUOkkwo6PxhbOVe96L0RUre7SfStaVcqhE/IbU= github.com/ViBiOh/exas v0.5.0/go.mod h1:sKRV/vz85Vj0caT+a3XWVFp0gyzXe36eZWL+XKge93E= github.com/ViBiOh/flags v1.2.0 h1:DaujjNXzD29KxKyp4eZdn7c9+uBN5DokWgDAe7DcUmc= github.com/ViBiOh/flags v1.2.0/go.mod h1:UyMB5zeD/aId7Xw3x7577ZNU298JmukzOcV8p/H2W1s= -github.com/ViBiOh/httputils/v4 v4.47.0 h1:kXqVTt16sHhOm52YR00DDtwYvwsr8/WiHhxo+WgiHtc= -github.com/ViBiOh/httputils/v4 v4.47.0/go.mod h1:ghw8fNnYr+1450CNeTyaDreaMDC7plFutd54ttT3EMs= +github.com/ViBiOh/httputils/v4 v4.47.1 h1:OPepRC9XVqsUGsDPL1P5tP8zqrACXSg5u0HSwB1wd+I= +github.com/ViBiOh/httputils/v4 v4.47.1/go.mod h1:ghw8fNnYr+1450CNeTyaDreaMDC7plFutd54ttT3EMs= github.com/ViBiOh/vith v0.5.0 h1:BpmjwWxd4NGmUJF12XpFOngK32539Lw3mvbMOaeOmhE= github.com/ViBiOh/vith v0.5.0/go.mod h1:Wf40eedOBEcxeLzGzGpuO+2G8qe5Ov1WIIBPXohyKco= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/pkg/exif/aggregate.go b/pkg/exif/aggregate.go index b98b4fe3..f2d15ce9 100644 --- a/pkg/exif/aggregate.go +++ b/pkg/exif/aggregate.go @@ -20,8 +20,8 @@ var ( levels = []string{"city", "state", "country"} ) -func redisKey(itemID string) string { - return version.Redis("exif:" + itemID) +func redisKey(item absto.Item) string { + return version.Redis("exif:" + item.ID) } func (a App) GetExifFor(ctx context.Context, item absto.Item) (exas.Exif, error) { @@ -32,14 +32,7 @@ func (a App) GetExifFor(ctx context.Context, item absto.Item) (exas.Exif, error) ctx, end := tracer.StartSpan(ctx, a.tracer, "get_exif") defer end() - return cache.Retrieve(ctx, a.redisClient, func(ctx context.Context) (exas.Exif, error) { - exif, err := a.loadExif(ctx, item) - if err != nil && !absto.IsNotExist(err) { - return exif, fmt.Errorf("load exif: %w", err) - } - - return exif, nil - }, cacheDuration, redisKey(item.ID)) + return a.exifCacheApp.Get(ctx, item) } func (a App) GetAggregateFor(ctx context.Context, item absto.Item) (provider.Aggregate, error) { @@ -50,22 +43,15 @@ func (a App) GetAggregateFor(ctx context.Context, item absto.Item) (provider.Agg ctx, end := tracer.StartSpan(ctx, a.tracer, "aggregate") defer end() - return cache.Retrieve(ctx, a.redisClient, func(ctx context.Context) (provider.Aggregate, error) { - aggregate, err := a.loadAggregate(ctx, item) - if err != nil && !absto.IsNotExist(err) { - return aggregate, fmt.Errorf("load aggregate: %w", err) - } - - return aggregate, nil - }, cacheDuration, redisKey(item.ID)) + return a.aggregateCacheApp.Get(ctx, item) } func (a App) SaveExifFor(ctx context.Context, item absto.Item, exif exas.Exif) error { - return cache.EvictOnSuccess(ctx, a.redisClient, redisKey(item.ID), a.saveMetadata(ctx, item, exif)) + return cache.EvictOnSuccess(ctx, a.redisClient, redisKey(item), a.saveMetadata(ctx, item, exif)) } func (a App) SaveAggregateFor(ctx context.Context, item absto.Item, aggregate provider.Aggregate) error { - return cache.EvictOnSuccess(ctx, a.redisClient, redisKey(item.ID), a.saveMetadata(ctx, item, aggregate)) + return cache.EvictOnSuccess(ctx, a.redisClient, redisKey(item), a.saveMetadata(ctx, item, aggregate)) } func (a App) aggregate(ctx context.Context, item absto.Item) error { diff --git a/pkg/exif/event.go b/pkg/exif/event.go index 5b201bab..9030c3d9 100644 --- a/pkg/exif/event.go +++ b/pkg/exif/event.go @@ -49,7 +49,7 @@ func (a App) Rename(ctx context.Context, old, new absto.Item) error { return fmt.Errorf("rename exif: %w", err) } - if err := a.redisClient.Delete(ctx, redisKey(old.ID)); err != nil { + if err := a.redisClient.Delete(ctx, redisKey(old)); err != nil { return fmt.Errorf("cache: %s", err) } @@ -64,7 +64,7 @@ func (a App) handleStartEvent(ctx context.Context, event provider.Event) error { forced := event.IsForcedFor("exif") if event.GetMetadata("force") == "cache" { - if err := a.redisClient.Delete(ctx, redisKey(event.Item.ID)); err != nil { + if err := a.redisClient.Delete(ctx, redisKey(event.Item)); err != nil { logger.WithField("fn", "exif.startEvent").WithField("item", event.Item.Pathname).Error("flush cache: %s", err) } @@ -159,7 +159,7 @@ func (a App) delete(ctx context.Context, item absto.Item) error { return fmt.Errorf("delete: %w", err) } - if err := a.redisClient.Delete(ctx, redisKey(item.ID)); err != nil { + if err := a.redisClient.Delete(ctx, redisKey(item)); err != nil { return fmt.Errorf("cache: %s", err) } diff --git a/pkg/exif/exif.go b/pkg/exif/exif.go index f45cfc32..83f308b7 100644 --- a/pkg/exif/exif.go +++ b/pkg/exif/exif.go @@ -12,6 +12,7 @@ import ( "github.com/ViBiOh/fibr/pkg/provider" "github.com/ViBiOh/flags" amqpclient "github.com/ViBiOh/httputils/v4/pkg/amqp" + "github.com/ViBiOh/httputils/v4/pkg/cache" "github.com/ViBiOh/httputils/v4/pkg/httpjson" "github.com/ViBiOh/httputils/v4/pkg/logger" prom "github.com/ViBiOh/httputils/v4/pkg/prometheus" @@ -22,11 +23,13 @@ import ( ) type App struct { - tracer trace.Tracer - storageApp absto.Storage - listStorageApp absto.Storage - exifMetric *prometheus.CounterVec - aggregateMetric *prometheus.CounterVec + tracer trace.Tracer + storageApp absto.Storage + listStorageApp absto.Storage + exifMetric *prometheus.CounterVec + aggregateMetric *prometheus.CounterVec + exifCacheApp cache.App[absto.Item, exas.Exif] + aggregateCacheApp cache.App[absto.Item, provider.Aggregate] redisClient redis.App @@ -76,7 +79,7 @@ func New(config Config, storageApp absto.Storage, prometheusRegisterer prometheu } } - return App{ + app := App{ exifRequest: request.New().URL(strings.TrimSpace(*config.exifURL)).BasicAuth(strings.TrimSpace(*config.exifUser), *config.exifPass), directAccess: *config.directAccess, maxSize: int64(*config.maxSize), @@ -95,7 +98,12 @@ func New(config Config, storageApp absto.Storage, prometheusRegisterer prometheu exifMetric: prom.CounterVec(prometheusRegisterer, "fibr", "exif", "item", "state"), aggregateMetric: prom.CounterVec(prometheusRegisterer, "fibr", "aggregate", "item", "state"), - }, nil + } + + app.exifCacheApp = cache.New(redisClient, redisKey, app.loadExif, cacheDuration) + app.aggregateCacheApp = cache.New(redisClient, redisKey, app.loadAggregate, cacheDuration) + + return app, nil } func (a App) ListDir(ctx context.Context, item absto.Item) ([]absto.Item, error) { diff --git a/pkg/thumbnail/thumbnail.go b/pkg/thumbnail/thumbnail.go index 8f37c398..b5cff95e 100644 --- a/pkg/thumbnail/thumbnail.go +++ b/pkg/thumbnail/thumbnail.go @@ -15,6 +15,7 @@ import ( "github.com/ViBiOh/fibr/pkg/provider" "github.com/ViBiOh/flags" "github.com/ViBiOh/httputils/v4/pkg/amqp" + "github.com/ViBiOh/httputils/v4/pkg/cache" "github.com/ViBiOh/httputils/v4/pkg/httperror" "github.com/ViBiOh/httputils/v4/pkg/logger" prom "github.com/ViBiOh/httputils/v4/pkg/prometheus" @@ -42,6 +43,8 @@ type App struct { pathnameInput chan absto.Item metric *prometheus.CounterVec + cacheApp cache.App[string, absto.Item] + amqpClient *amqp.Client amqpThumbnailRoutingKey string amqpExchange string @@ -107,7 +110,7 @@ func New(config Config, storage absto.Storage, redisClient redis.App, prometheus sizes = []uint64{SmallSize} } - return App{ + app := App{ vithRequest: request.New().URL(*config.vithURL).BasicAuth(*config.vithUser, *config.vithPass).WithClient(provider.SlowClient), maxSize: *config.maxSize, @@ -134,7 +137,11 @@ func New(config Config, storage absto.Storage, redisClient redis.App, prometheus largeSize: largeSize, sizes: sizes, - }, nil + } + + app.cacheApp = cache.New(redisClient, redisKey, app.storageApp.Info, redisCacheDuration) + + return app, nil } // LargeThumbnailSize give large thumbnail size diff --git a/pkg/thumbnail/utils.go b/pkg/thumbnail/utils.go index 8392f178..c4c10d2f 100644 --- a/pkg/thumbnail/utils.go +++ b/pkg/thumbnail/utils.go @@ -8,7 +8,6 @@ import ( absto "github.com/ViBiOh/absto/pkg/model" "github.com/ViBiOh/fibr/pkg/provider" "github.com/ViBiOh/fibr/pkg/version" - "github.com/ViBiOh/httputils/v4/pkg/cache" "github.com/ViBiOh/httputils/v4/pkg/sha" "github.com/ViBiOh/httputils/v4/pkg/tracer" "github.com/ViBiOh/vith/pkg/model" @@ -89,7 +88,5 @@ func (a App) Info(ctx context.Context, pathname string) (absto.Item, error) { ctx, end := tracer.StartSpan(ctx, a.tracer, "info") defer end() - return cache.Retrieve(ctx, a.redisClient, func(ctx context.Context) (absto.Item, error) { - return a.storageApp.Info(ctx, pathname) - }, redisCacheDuration, redisKey(pathname)) + return a.cacheApp.Get(ctx, pathname) } diff --git a/pkg/thumbnail/utils_test.go b/pkg/thumbnail/utils_test.go index 83200b4b..edc227fc 100644 --- a/pkg/thumbnail/utils_test.go +++ b/pkg/thumbnail/utils_test.go @@ -6,6 +6,7 @@ import ( absto "github.com/ViBiOh/absto/pkg/model" "github.com/ViBiOh/fibr/pkg/mocks" + "github.com/ViBiOh/httputils/v4/pkg/cache" "github.com/golang/mock/gomock" ) @@ -86,6 +87,7 @@ func TestHasThumbnail(t *testing.T) { storageMock := mocks.NewStorage(ctrl) tc.instance.storageApp = storageMock + tc.instance.cacheApp = cache.New(nil, nil, storageMock.Info, 0) if intention == "found" { storageMock.EXPECT().Info(gomock.Any(), gomock.Any()).Return(absto.Item{}, nil)