Skip to content

Commit

Permalink
refactor: Using cache layer more efficiently
Browse files Browse the repository at this point in the history 
Signed-off-by: Vincent Boutour <[email protected]>
  • Loading branch information
@ViBiOh
ViBiOh committed Sep 3, 2022
1 parent 5e84177 commit ac5bb50
Show file tree
Hide file tree
Showing 8 changed files with 45 additions and 45 deletions.
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ module github.com/ViBiOh/fibr
go 1.19

require (
github.com/ViBiOh/ChatPotte v0.2.5
github.com/ViBiOh/ChatPotte v0.2.6
github.com/ViBiOh/absto v1.0.3
github.com/ViBiOh/auth/v2 v2.14.2
github.com/ViBiOh/auth/v2 v2.14.3
github.com/ViBiOh/exas v0.5.0
github.com/ViBiOh/flags v1.2.0
github.com/ViBiOh/httputils/v4 v4.47.0
github.com/ViBiOh/httputils/v4 v4.47.1
github.com/ViBiOh/vith v0.5.0
github.com/golang/mock v1.6.0
github.com/prometheus/client_golang v1.13.0
Expand Down
12 changes: 6 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,18 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/ViBiOh/ChatPotte v0.2.5 h1:nZrC0RN5RT0DRGHlpVUhhOWUPFeLKq18P35I9ZMacuM=
github.com/ViBiOh/ChatPotte v0.2.5/go.mod h1:H/B3xIv0A3eddp61mlENHg7P0DEu3hJJnVdD6bRkzhs=
github.com/ViBiOh/ChatPotte v0.2.6 h1:7neA/C6vEPlnNW2ymlWQf2wmp8WUSQsJz/2iV18l18M=
github.com/ViBiOh/ChatPotte v0.2.6/go.mod h1:hGY0JXNxQOLa1Lc7G7j3xpN3PltI9MjpXoI3dWR8e5s=
github.com/ViBiOh/absto v1.0.3 h1:guwVdRN7mIJM6XO65ABqDjz/7JiFMGJ7+VfmE/Ni3Zw=
github.com/ViBiOh/absto v1.0.3/go.mod h1:nu8N6LQnyxYhdnTAOtlpr4zca2tOcj5uUgI30oCF4vo=
github.com/ViBiOh/auth/v2 v2.14.2 h1:3z89LtCGs8zZFLOTzHiN3YIacZ2UDxzpkQ/sO2XJcfY=
github.com/ViBiOh/auth/v2 v2.14.2/go.mod h1:wktrqt4O8toixNWekhJ6cL8cXXMOLCSlKBhWUIpHDto=
github.com/ViBiOh/auth/v2 v2.14.3 h1:PHkryZKMG/CRNAqgM7yxXzLI9GRYytxX8OaO7XVsk4E=
github.com/ViBiOh/auth/v2 v2.14.3/go.mod h1:+gm2mDOj/178egUVCqrug7tOwI32IkZsofT1YaL+jJY=
github.com/ViBiOh/exas v0.5.0 h1:ZL5v5rUOkkwo6PxhbOVe96L0RUre7SfStaVcqhE/IbU=
github.com/ViBiOh/exas v0.5.0/go.mod h1:sKRV/vz85Vj0caT+a3XWVFp0gyzXe36eZWL+XKge93E=
github.com/ViBiOh/flags v1.2.0 h1:DaujjNXzD29KxKyp4eZdn7c9+uBN5DokWgDAe7DcUmc=
github.com/ViBiOh/flags v1.2.0/go.mod h1:UyMB5zeD/aId7Xw3x7577ZNU298JmukzOcV8p/H2W1s=
github.com/ViBiOh/httputils/v4 v4.47.0 h1:kXqVTt16sHhOm52YR00DDtwYvwsr8/WiHhxo+WgiHtc=
github.com/ViBiOh/httputils/v4 v4.47.0/go.mod h1:ghw8fNnYr+1450CNeTyaDreaMDC7plFutd54ttT3EMs=
github.com/ViBiOh/httputils/v4 v4.47.1 h1:OPepRC9XVqsUGsDPL1P5tP8zqrACXSg5u0HSwB1wd+I=
github.com/ViBiOh/httputils/v4 v4.47.1/go.mod h1:ghw8fNnYr+1450CNeTyaDreaMDC7plFutd54ttT3EMs=
github.com/ViBiOh/vith v0.5.0 h1:BpmjwWxd4NGmUJF12XpFOngK32539Lw3mvbMOaeOmhE=
github.com/ViBiOh/vith v0.5.0/go.mod h1:Wf40eedOBEcxeLzGzGpuO+2G8qe5Ov1WIIBPXohyKco=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
Expand Down
26 changes: 6 additions & 20 deletions pkg/exif/aggregate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ var (
levels = []string{"city", "state", "country"}
)

func redisKey(itemID string) string {
return version.Redis("exif:" + itemID)
func redisKey(item absto.Item) string {
return version.Redis("exif:" + item.ID)
}

func (a App) GetExifFor(ctx context.Context, item absto.Item) (exas.Exif, error) {
Expand All @@ -32,14 +32,7 @@ func (a App) GetExifFor(ctx context.Context, item absto.Item) (exas.Exif, error)
ctx, end := tracer.StartSpan(ctx, a.tracer, "get_exif")
defer end()

return cache.Retrieve(ctx, a.redisClient, func(ctx context.Context) (exas.Exif, error) {
exif, err := a.loadExif(ctx, item)
if err != nil && !absto.IsNotExist(err) {
return exif, fmt.Errorf("load exif: %w", err)
}

return exif, nil
}, cacheDuration, redisKey(item.ID))
return a.exifCacheApp.Get(ctx, item)
}

func (a App) GetAggregateFor(ctx context.Context, item absto.Item) (provider.Aggregate, error) {
Expand All @@ -50,22 +43,15 @@ func (a App) GetAggregateFor(ctx context.Context, item absto.Item) (provider.Agg
ctx, end := tracer.StartSpan(ctx, a.tracer, "aggregate")
defer end()

return cache.Retrieve(ctx, a.redisClient, func(ctx context.Context) (provider.Aggregate, error) {
aggregate, err := a.loadAggregate(ctx, item)
if err != nil && !absto.IsNotExist(err) {
return aggregate, fmt.Errorf("load aggregate: %w", err)
}

return aggregate, nil
}, cacheDuration, redisKey(item.ID))
return a.aggregateCacheApp.Get(ctx, item)
}

func (a App) SaveExifFor(ctx context.Context, item absto.Item, exif exas.Exif) error {
return cache.EvictOnSuccess(ctx, a.redisClient, redisKey(item.ID), a.saveMetadata(ctx, item, exif))
return cache.EvictOnSuccess(ctx, a.redisClient, redisKey(item), a.saveMetadata(ctx, item, exif))
}

func (a App) SaveAggregateFor(ctx context.Context, item absto.Item, aggregate provider.Aggregate) error {
return cache.EvictOnSuccess(ctx, a.redisClient, redisKey(item.ID), a.saveMetadata(ctx, item, aggregate))
return cache.EvictOnSuccess(ctx, a.redisClient, redisKey(item), a.saveMetadata(ctx, item, aggregate))
}

func (a App) aggregate(ctx context.Context, item absto.Item) error {
Expand Down
6 changes: 3 additions & 3 deletions pkg/exif/event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (a App) Rename(ctx context.Context, old, new absto.Item) error {
return fmt.Errorf("rename exif: %w", err)
}

if err := a.redisClient.Delete(ctx, redisKey(old.ID)); err != nil {
if err := a.redisClient.Delete(ctx, redisKey(old)); err != nil {
return fmt.Errorf("cache: %s", err)
}

Expand All @@ -64,7 +64,7 @@ func (a App) handleStartEvent(ctx context.Context, event provider.Event) error {
forced := event.IsForcedFor("exif")

if event.GetMetadata("force") == "cache" {
if err := a.redisClient.Delete(ctx, redisKey(event.Item.ID)); err != nil {
if err := a.redisClient.Delete(ctx, redisKey(event.Item)); err != nil {
logger.WithField("fn", "exif.startEvent").WithField("item", event.Item.Pathname).Error("flush cache: %s", err)
}

Expand Down Expand Up @@ -159,7 +159,7 @@ func (a App) delete(ctx context.Context, item absto.Item) error {
return fmt.Errorf("delete: %w", err)
}

if err := a.redisClient.Delete(ctx, redisKey(item.ID)); err != nil {
if err := a.redisClient.Delete(ctx, redisKey(item)); err != nil {
return fmt.Errorf("cache: %s", err)
}

Expand Down
22 changes: 15 additions & 7 deletions pkg/exif/exif.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/ViBiOh/fibr/pkg/provider"
"github.com/ViBiOh/flags"
amqpclient "github.com/ViBiOh/httputils/v4/pkg/amqp"
"github.com/ViBiOh/httputils/v4/pkg/cache"
"github.com/ViBiOh/httputils/v4/pkg/httpjson"
"github.com/ViBiOh/httputils/v4/pkg/logger"
prom "github.com/ViBiOh/httputils/v4/pkg/prometheus"
Expand All @@ -22,11 +23,13 @@ import (
)

type App struct {
tracer trace.Tracer
storageApp absto.Storage
listStorageApp absto.Storage
exifMetric *prometheus.CounterVec
aggregateMetric *prometheus.CounterVec
tracer trace.Tracer
storageApp absto.Storage
listStorageApp absto.Storage
exifMetric *prometheus.CounterVec
aggregateMetric *prometheus.CounterVec
exifCacheApp cache.App[absto.Item, exas.Exif]
aggregateCacheApp cache.App[absto.Item, provider.Aggregate]

redisClient redis.App

Expand Down Expand Up @@ -76,7 +79,7 @@ func New(config Config, storageApp absto.Storage, prometheusRegisterer prometheu
}
}

return App{
app := App{
exifRequest: request.New().URL(strings.TrimSpace(*config.exifURL)).BasicAuth(strings.TrimSpace(*config.exifUser), *config.exifPass),
directAccess: *config.directAccess,
maxSize: int64(*config.maxSize),
Expand All @@ -95,7 +98,12 @@ func New(config Config, storageApp absto.Storage, prometheusRegisterer prometheu

exifMetric: prom.CounterVec(prometheusRegisterer, "fibr", "exif", "item", "state"),
aggregateMetric: prom.CounterVec(prometheusRegisterer, "fibr", "aggregate", "item", "state"),
}, nil
}

app.exifCacheApp = cache.New(redisClient, redisKey, app.loadExif, cacheDuration)
app.aggregateCacheApp = cache.New(redisClient, redisKey, app.loadAggregate, cacheDuration)

return app, nil
}

func (a App) ListDir(ctx context.Context, item absto.Item) ([]absto.Item, error) {
Expand Down
11 changes: 9 additions & 2 deletions pkg/thumbnail/thumbnail.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/ViBiOh/fibr/pkg/provider"
"github.com/ViBiOh/flags"
"github.com/ViBiOh/httputils/v4/pkg/amqp"
"github.com/ViBiOh/httputils/v4/pkg/cache"
"github.com/ViBiOh/httputils/v4/pkg/httperror"
"github.com/ViBiOh/httputils/v4/pkg/logger"
prom "github.com/ViBiOh/httputils/v4/pkg/prometheus"
Expand Down Expand Up @@ -42,6 +43,8 @@ type App struct {
pathnameInput chan absto.Item
metric *prometheus.CounterVec

cacheApp cache.App[string, absto.Item]

amqpClient *amqp.Client
amqpThumbnailRoutingKey string
amqpExchange string
Expand Down Expand Up @@ -107,7 +110,7 @@ func New(config Config, storage absto.Storage, redisClient redis.App, prometheus
sizes = []uint64{SmallSize}
}

return App{
app := App{
vithRequest: request.New().URL(*config.vithURL).BasicAuth(*config.vithUser, *config.vithPass).WithClient(provider.SlowClient),

maxSize: *config.maxSize,
Expand All @@ -134,7 +137,11 @@ func New(config Config, storage absto.Storage, redisClient redis.App, prometheus

largeSize: largeSize,
sizes: sizes,
}, nil
}

app.cacheApp = cache.New(redisClient, redisKey, app.storageApp.Info, redisCacheDuration)

return app, nil
}

// LargeThumbnailSize give large thumbnail size
Expand Down
5 changes: 1 addition & 4 deletions pkg/thumbnail/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
absto "github.com/ViBiOh/absto/pkg/model"
"github.com/ViBiOh/fibr/pkg/provider"
"github.com/ViBiOh/fibr/pkg/version"
"github.com/ViBiOh/httputils/v4/pkg/cache"
"github.com/ViBiOh/httputils/v4/pkg/sha"
"github.com/ViBiOh/httputils/v4/pkg/tracer"
"github.com/ViBiOh/vith/pkg/model"
Expand Down Expand Up @@ -89,7 +88,5 @@ func (a App) Info(ctx context.Context, pathname string) (absto.Item, error) {
ctx, end := tracer.StartSpan(ctx, a.tracer, "info")
defer end()

return cache.Retrieve(ctx, a.redisClient, func(ctx context.Context) (absto.Item, error) {
return a.storageApp.Info(ctx, pathname)
}, redisCacheDuration, redisKey(pathname))
return a.cacheApp.Get(ctx, pathname)
}
2 changes: 2 additions & 0 deletions pkg/thumbnail/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

absto "github.com/ViBiOh/absto/pkg/model"
"github.com/ViBiOh/fibr/pkg/mocks"
"github.com/ViBiOh/httputils/v4/pkg/cache"
"github.com/golang/mock/gomock"
)

Expand Down Expand Up @@ -86,6 +87,7 @@ func TestHasThumbnail(t *testing.T) {
storageMock := mocks.NewStorage(ctrl)

tc.instance.storageApp = storageMock
tc.instance.cacheApp = cache.New(nil, nil, storageMock.Info, 0)

if intention == "found" {
storageMock.EXPECT().Info(gomock.Any(), gomock.Any()).Return(absto.Item{}, nil)
Expand Down

0 comments on commit ac5bb50

Please sign in to comment.