feat: Adding redis cache for thumbnail info

Signed-off-by: Vincent Boutour <[email protected]>

cmd/fibr/fibr.go

@@ -133,7 +133,7 @@ func main() {
 
 	redisClient := redis.New(redisConfig, prometheusApp.Registerer(), tracerApp.GetTracer("redis"))
 
-	thumbnailApp, err := thumbnail.New(thumbnailConfig, storageProvider, prometheusRegisterer, amqpClient)
+	thumbnailApp, err := thumbnail.New(thumbnailConfig, storageProvider, redisClient, prometheusRegisterer, tracerApp.GetTracer("thumbnail"), amqpClient)
 	logger.Fatal(err)
 
 	rendererApp, err := renderer.New(rendererConfig, content, fibr.FuncMap, tracerApp.GetTracer("renderer"))

cmd/fibr/templates/stats.html

@@ -41,6 +41,14 @@ <h2 class="center">Stats</h2>
       </p>
     </form>
 
+    <form method="post" action="#">
+      <input type="hidden" name="method" value="TRACE" />
+      <input type="hidden" name="subset" value="cache" />
+      <p class="padding no-margin center">
+        <button type="submit" class="button bg-danger">Flush Redis cache</button>
+      </p>
+    </form>
+
     <form method="post" action="#">
       <input type="hidden" name="method" value="TRACE" />
       <input type="hidden" name="subset" value="all" />

pkg/exif/aggregate.go

@@ -8,8 +8,8 @@ import (
 	absto "github.com/ViBiOh/absto/pkg/model"
 	exas "github.com/ViBiOh/exas/pkg/model"
 	"github.com/ViBiOh/fibr/pkg/provider"
+	"github.com/ViBiOh/fibr/pkg/version"
 	"github.com/ViBiOh/httputils/v4/pkg/cache"
-	"github.com/ViBiOh/httputils/v4/pkg/sha"
 	"github.com/ViBiOh/httputils/v4/pkg/tracer"
 )
 
@@ -18,12 +18,10 @@ var (
 	aggregateRatio = 0.4
 
 	levels = []string{"city", "state", "country"}
-
-	cacheVersion = sha.New("vibioh/fibr/1")[:8]
 )
 
 func redisKey(itemID string) string {
-	return fmt.Sprintf("fibr:%s:exif:%s", cacheVersion, itemID)
+	return version.Redis("exif:" + itemID)
 }
 
 func (a App) GetExifFor(ctx context.Context, item absto.Item) (exas.Exif, error) {

pkg/exif/event.go

@@ -59,6 +59,16 @@ func getEventLogger(item absto.Item) logger.Provider {
 func (a App) handleStartEvent(ctx context.Context, event provider.Event) error {
 	forced := event.IsForcedFor("exif")
 
+	if event.GetMetadata("force") == "cache" {
+		if err := a.redisClient.Delete(ctx, redisKey(event.Item.ID)); err != nil {
+			logger.WithField("fn", "exif.startEvent").WithField("item", event.Item.Pathname).Error("flush cache: %s", err)
+		}
+
+		if !forced {
+			return nil
+		}
+	}
+
 	item := event.Item
 	if !forced && a.hasMetadata(ctx, item) {
 		logger.WithField("item", item.Pathname).Debug("has metadata")

pkg/thumbnail/event.go

@@ -6,6 +6,7 @@ import (
 
 	absto "github.com/ViBiOh/absto/pkg/model"
 	"github.com/ViBiOh/fibr/pkg/provider"
+	"github.com/ViBiOh/httputils/v4/pkg/cache"
 	"github.com/ViBiOh/httputils/v4/pkg/logger"
 )
 
@@ -60,11 +61,23 @@ func (a App) generateItem(ctx context.Context, event provider.Event) {
 	forced := event.IsForcedFor("thumbnail")
 
 	for _, size := range a.sizes {
+		cacheKey := redisKey(a.PathForScale(event.Item, size))
+
+		if event.GetMetadata("force") == "cache" {
+			if err := a.redisClient.Delete(ctx, cacheKey); err != nil {
+				logger.WithField("fn", "thumbnail.generate").WithField("item", event.Item.Pathname).Error("flush cache for scale %d: %s", size, err)
+			}
+
+			if !forced {
+				continue
+			}
+		}
+
 		if !forced && a.HasThumbnail(ctx, event.Item, size) {
 			continue
 		}
 
-		if err := a.generate(ctx, event.Item, size); err != nil {
+		if err := cache.EvictOnSuccess(ctx, a.redisClient, cacheKey, a.generate(ctx, event.Item, size)); err != nil {
 			logger.WithField("fn", "thumbnail.generate").WithField("item", event.Item.Pathname).Error("generate for scale %d: %s", size, err)
 		}
 	}
@@ -78,7 +91,7 @@ func (a App) generateStreamIfNeeded(ctx context.Context, event provider.Event) {
 	if needStream, err := a.shouldGenerateStream(ctx, event.Item); err != nil {
 		logger.Error("determine if stream generation is possible: %s", err)
 	} else if needStream {
-		if err = a.generateStream(ctx, event.Item); err != nil {
+		if err = cache.EvictOnSuccess(ctx, a.redisClient, redisKey(getStreamPath(event.Item)), a.generateStream(ctx, event.Item)); err != nil {
 			logger.Error("generate stream: %s", err)
 		}
 	}

pkg/thumbnail/stream.go

@@ -16,7 +16,7 @@ import (
 
 // HasStream checks if given item has a streamable version
 func (a App) HasStream(ctx context.Context, item absto.Item) bool {
-	_, err := a.storageApp.Info(ctx, getStreamPath(item))
+	_, err := a.Info(ctx, getStreamPath(item))
 	return err == nil
 }
 

pkg/thumbnail/thumbnail.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"path"
 	"strings"
 	"time"
 
@@ -20,9 +19,11 @@ import (
 	"github.com/ViBiOh/httputils/v4/pkg/logger"
 	prom "github.com/ViBiOh/httputils/v4/pkg/prometheus"
 	"github.com/ViBiOh/httputils/v4/pkg/query"
+	"github.com/ViBiOh/httputils/v4/pkg/redis"
 	"github.com/ViBiOh/httputils/v4/pkg/request"
 	"github.com/ViBiOh/httputils/v4/pkg/sha"
 	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel/trace"
 )
 
 const (
@@ -33,6 +34,8 @@ const (
 var cacheDuration = fmt.Sprintf("private, max-age=%.0f", (time.Minute * 5).Seconds())
 
 type App struct {
+	redisClient     redis.App
+	tracer          trace.Tracer
 	storageApp      absto.Storage
 	smallStorageApp absto.Storage
 	largeStorageApp absto.Storage
@@ -86,7 +89,7 @@ func Flags(fs *flag.FlagSet, prefix string) Config {
 	}
 }
 
-func New(config Config, storage absto.Storage, prometheusRegisterer prometheus.Registerer, amqpClient *amqp.Client) (App, error) {
+func New(config Config, storage absto.Storage, redisClient redis.App, prometheusRegisterer prometheus.Registerer, tracer trace.Tracer, amqpClient *amqp.Client) (App, error) {
 	var amqpExchange string
 	if amqpClient != nil {
 		amqpExchange = strings.TrimSpace(*config.amqpExchange)
@@ -111,6 +114,9 @@ func New(config Config, storage absto.Storage, prometheusRegisterer prometheus.R
 		minBitrate:   *config.minBitrate,
 		directAccess: *config.directAccess,
 
+		redisClient: redisClient,
+		tracer:      tracer,
+
 		amqpExchange:            amqpExchange,
 		amqpStreamRoutingKey:    strings.TrimSpace(*config.amqpStreamRoutingKey),
 		amqpThumbnailRoutingKey: strings.TrimSpace(*config.amqpThumbnailRoutingKey),
@@ -171,24 +177,27 @@ func (a App) Serve(w http.ResponseWriter, r *http.Request, item absto.Item) {
 
 	ctx := r.Context()
 
-	thumbnailInfo, ok := a.ThumbnailInfo(ctx, item, scale)
-	if !ok {
-		w.WriteHeader(http.StatusNoContent)
-		return
-	}
+	name := a.PathForScale(item, scale)
 
-	reader, err := a.storageApp.ReadFrom(ctx, thumbnailInfo.Pathname)
+	reader, err := a.storageApp.ReadFrom(ctx, name)
 	if err != nil {
+		if absto.IsNotExist(err) {
+			w.WriteHeader(http.StatusNoContent)
+		}
+
 		httperror.InternalServerError(w, err)
+
 		return
 	}
 
+	baseName := name
+
 	defer provider.LogClose(reader, "thumbnail.Serve", item.Pathname)
 
 	w.Header().Add("Cache-Control", cacheDuration)
-	w.Header().Add("Content-Disposition", fmt.Sprintf("inline; filename=%s", path.Base(thumbnailInfo.Pathname)))
+	w.Header().Add("Content-Disposition", fmt.Sprintf("inline; filename=%s", baseName))
 
-	http.ServeContent(w, r, path.Base(thumbnailInfo.Pathname), item.Date, reader)
+	http.ServeContent(w, r, baseName, item.Date, reader)
 }
 
 // List return all thumbnails in a base64 form
@@ -238,7 +247,7 @@ func (a App) thumbnailHash(ctx context.Context, items []absto.Item) string {
 	hasher := sha.Stream()
 
 	for _, item := range items {
-		if info, err := a.storageApp.Info(ctx, a.PathForScale(item, SmallSize)); err == nil {
+		if info, err := a.Info(ctx, a.PathForScale(item, SmallSize)); err == nil {
 			hasher.Write(info)
 		}
 	}

pkg/thumbnail/utils.go

@@ -3,18 +3,23 @@ package thumbnail
 import (
 	"context"
 	"fmt"
+	"time"
 
 	absto "github.com/ViBiOh/absto/pkg/model"
 	"github.com/ViBiOh/fibr/pkg/provider"
+	"github.com/ViBiOh/fibr/pkg/version"
+	"github.com/ViBiOh/httputils/v4/pkg/cache"
+	"github.com/ViBiOh/httputils/v4/pkg/sha"
+	"github.com/ViBiOh/httputils/v4/pkg/tracer"
 	"github.com/ViBiOh/vith/pkg/model"
 )
 
-// CanHaveThumbnail determine if thumbnail can be generated for given pathname
+var redisCacheDuration = time.Hour * 96
+
 func (a App) CanHaveThumbnail(item absto.Item) bool {
 	return !item.IsDir && provider.ThumbnailExtensions[item.Extension] && (a.maxSize == 0 || item.Size < a.maxSize || a.directAccess)
 }
 
-// HasLargeThumbnail determine if large thumbnail exist for given pathname
 func (a App) HasLargeThumbnail(ctx context.Context, item absto.Item) bool {
 	if a.largeSize == 0 {
 		return false
@@ -23,36 +28,23 @@ func (a App) HasLargeThumbnail(ctx context.Context, item absto.Item) bool {
 	return a.HasThumbnail(ctx, item, a.largeSize)
 }
 
-// HasThumbnail determine if thumbnail exist for given pathname
 func (a App) HasThumbnail(ctx context.Context, item absto.Item, scale uint64) bool {
-	_, ok := a.ThumbnailInfo(ctx, item, scale)
-	return ok
-}
-
-// ThumbnailInfo determine if thumbnail exist for given pathname and provide detail about it
-func (a App) ThumbnailInfo(ctx context.Context, item absto.Item, scale uint64) (thumbnailItem absto.Item, ok bool) {
 	if item.IsDir {
-		ok = false
-		return
+		return false
 	}
 
-	var err error
-	thumbnailItem, err = a.storageApp.Info(ctx, a.PathForScale(item, scale))
-	ok = err == nil
-	return
+	_, err := a.Info(ctx, a.PathForScale(item, scale))
+	return err == nil
 }
 
-// Path computes thumbnail path for a a given item
 func (a App) Path(item absto.Item) string {
 	return a.PathForScale(item, SmallSize)
 }
 
-// PathForLarge computes thumbnail path for a a given item and large size
 func (a App) PathForLarge(item absto.Item) string {
 	return a.PathForScale(item, a.largeSize)
 }
 
-// PathForScale computes thumbnail path for a a given item and scale
 func (a App) PathForScale(item absto.Item, scale uint64) string {
 	if item.IsDir {
 		return provider.MetadataDirectory(item)
@@ -66,16 +58,14 @@ func (a App) PathForScale(item absto.Item, scale uint64) string {
 	return getThumbnailPathForExtension(item, "webp")
 }
 
-// GetChunk retrieve the storage item in the metadata
 func (a App) GetChunk(ctx context.Context, pathname string) (absto.Item, error) {
-	return a.storageApp.Info(ctx, provider.MetadataDirectoryName+pathname)
+	return a.Info(ctx, provider.MetadataDirectoryName+pathname)
 }
 
 func getStreamPath(item absto.Item) string {
 	return getThumbnailPathForExtension(item, "m3u8")
 }
 
-// PathWithExtension computes thumbnail path with given extension
 func getThumbnailPathForExtension(item absto.Item, extension string) string {
 	return fmt.Sprintf("%s%s.%s", provider.MetadataDirectory(item), item.ID, extension)
 }
@@ -90,3 +80,16 @@ func typeOfItem(item absto.Item) model.ItemType {
 
 	return itemType
 }
+
+func redisKey(id string) string {
+	return version.Redis("thumbnail:" + sha.New(id))
+}
+
+func (a App) Info(ctx context.Context, pathname string) (absto.Item, error) {
+	ctx, end := tracer.StartSpan(ctx, a.tracer, "info")
+	defer end()
+
+	return cache.Retrieve(ctx, a.redisClient, redisKey(pathname), func(ctx context.Context) (absto.Item, error) {
+		return a.storageApp.Info(ctx, pathname)
+	}, redisCacheDuration)
+}

pkg/version/version.go

@@ -0,0 +1,13 @@
+package version
+
+import (
+	"fmt"
+
+	"github.com/ViBiOh/httputils/v4/pkg/sha"
+)
+
+var cacheVersion = sha.New("vibioh/fibr/1")[:8]
+
+func Redis(content string) string {
+	return fmt.Sprintf("fibr:%s:%s", cacheVersion, content)
+}