feat(database): Adding badger database

Signed-off-by: Vincent Boutour <[email protected]>

cmd/fibr/fibr.go

@@ -9,6 +9,7 @@ import (
 	authMiddleware "github.com/ViBiOh/auth/v2/pkg/middleware"
 	basicMemory "github.com/ViBiOh/auth/v2/pkg/store/memory"
 	"github.com/ViBiOh/fibr/pkg/crud"
+	"github.com/ViBiOh/fibr/pkg/database"
 	"github.com/ViBiOh/fibr/pkg/exif"
 	"github.com/ViBiOh/fibr/pkg/fibr"
 	"github.com/ViBiOh/fibr/pkg/filesystem"
@@ -75,6 +76,14 @@ func main() {
 	storageApp, err := filesystem.New(filesystemConfig)
 	logger.Fatal(err)
 
+	databaseApp, err := database.New(storageApp)
+	logger.Fatal(err)
+	defer func() {
+		if err := databaseApp.Close(); err != nil {
+			logger.Error("unable to close database: %s", err)
+		}
+	}()
+
 	prometheusRegister := prometheusApp.Registerer()
 
 	thumbnailApp := thumbnail.New(thumbnailConfig, storageApp, prometheusRegister)
@@ -84,7 +93,7 @@ func main() {
 	logger.Fatal(err)
 
 	metadataApp := metadata.New(metadataConfig, storageApp)
-	crudApp, err := crud.New(crudConfig, storageApp, rendererApp, metadataApp, thumbnailApp, exifApp, prometheusRegister)
+	crudApp, err := crud.New(crudConfig, storageApp, rendererApp, metadataApp, thumbnailApp, exifApp, databaseApp, prometheusRegister)
 	logger.Fatal(err)
 
 	var middlewareApp authMiddleware.App

go.mod

@@ -5,6 +5,7 @@ go 1.16
 require (
 	github.com/ViBiOh/auth/v2 v2.9.2
 	github.com/ViBiOh/httputils/v4 v4.17.1
+	github.com/dgraph-io/badger/v3 v3.2103.1
 	github.com/golang/mock v1.6.0
 	github.com/prometheus/client_golang v1.11.0
 	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a

infra/web.yaml

@@ -23,6 +23,8 @@ spec:
     cleanupOnFail: true
   values:
     nameOverride: fibr
+    strategy:
+      type: Recreate
     image:
       name: vibioh/fibr
       tag: '202107241806' # {"$imagepolicy": "default:flux-fibr-web:tag"}

pkg/crud/crud.go

@@ -1,6 +1,7 @@
 package crud
 
 import (
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -10,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ViBiOh/fibr/pkg/database"
 	"github.com/ViBiOh/fibr/pkg/exif"
 	"github.com/ViBiOh/fibr/pkg/metadata"
 	"github.com/ViBiOh/fibr/pkg/provider"
@@ -70,6 +72,7 @@ type app struct {
 	metadataApp  metadata.App
 	thumbnailApp thumbnail.App
 	exifApp      exif.App
+	databaseApp  database.App
 
 	sanitizeOnStart bool
 	exifDateOnStart bool
@@ -85,7 +88,7 @@ func Flags(fs *flag.FlagSet, prefix string) Config {
 }
 
 // New creates new App from Config
-func New(config Config, storage provider.Storage, rendererApp renderer.App, metadataApp metadata.App, thumbnailApp thumbnail.App, exifApp exif.App, prometheus prometheus.Registerer) (App, error) {
+func New(config Config, storage provider.Storage, rendererApp renderer.App, metadataApp metadata.App, thumbnailApp thumbnail.App, exifApp exif.App, databaseApp database.App, prometheus prometheus.Registerer) (App, error) {
 	app := &app{
 		sanitizeOnStart: *config.sanitizeOnStart,
 		exifDateOnStart: *config.exifDateOnStart,
@@ -94,6 +97,7 @@ func New(config Config, storage provider.Storage, rendererApp renderer.App, meta
 		rendererApp:  rendererApp,
 		thumbnailApp: thumbnailApp,
 		exifApp:      exifApp,
+		databaseApp:  databaseApp,
 		metadataApp:  metadataApp,
 		prometheus:   prometheus,
 	}
@@ -146,9 +150,17 @@ func (a *app) Start(done <-chan struct{}) {
 			a.thumbnailApp.GenerateThumbnail(item)
 		}
 
-		if a.exifDateOnStart && exif.CanHaveExif(item) {
-			if err := a.dateFromExif(item); err != nil {
-				logger.Warn("unable to update date from exif for `%s`: %s", item.Pathname, err)
+		if exif.CanHaveExif(item) {
+			if !a.databaseApp.HasEntry(item) {
+				if err := a.saveExif(item); err != nil {
+					logger.Error("unable to save exif for `%s`: %s", item.Pathname, err)
+				}
+			}
+
+			if a.exifDateOnStart && a.databaseApp.HasEntry(item) {
+				if err := a.dateFromExif(item); err != nil {
+					logger.Warn("unable to update date from exif for `%s`: %s", item.Pathname, err)
+				}
 			}
 		}
 
@@ -193,12 +205,30 @@ func (a *app) rename(item provider.StorageItem, name string, guage *prometheus.G
 	return renamedItem
 }
 
-func (a *app) dateFromExif(item provider.StorageItem) error {
+func (a *app) saveExif(item provider.StorageItem) error {
 	data, err := a.exifApp.Get(item)
 	if err != nil {
 		return fmt.Errorf("unable to get exif data: %s", err)
 	}
 
+	payload, err := json.Marshal(data)
+	if err != nil {
+		return fmt.Errorf("unable to marshal exif data: %s", err)
+	}
+
+	if err := a.databaseApp.Store([]byte(item.Pathname), payload); err != nil {
+		return fmt.Errorf("unable to store exif data: %s", err)
+	}
+
+	return nil
+}
+
+func (a *app) dateFromExif(item provider.StorageItem) error {
+	data, err := a.databaseApp.Get(item)
+	if err != nil {
+		return fmt.Errorf("unable to retrieve exif data: %s", err)
+	}
+
 	rawCreateDate, ok := data[exifDate]
 	if !ok {
 		return fmt.Errorf("no `%s` found", exifDate)

pkg/database/database.go

@@ -0,0 +1,76 @@
+package database
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+
+	"github.com/ViBiOh/fibr/pkg/provider"
+	badger "github.com/dgraph-io/badger/v3"
+)
+
+// App of package
+type App interface {
+	Get(provider.StorageItem) (map[string]interface{}, error)
+	HasEntry(provider.StorageItem) bool
+	Store([]byte, []byte) error
+	Close() error
+}
+
+type app struct {
+	db *badger.DB
+}
+
+// New creates new App
+func New(storageApp provider.Storage) (App, error) {
+	db, err := badger.Open(badger.DefaultOptions(storageApp.Path(path.Join(provider.MetadataDirectoryName, "database"))))
+	if err != nil {
+		return nil, fmt.Errorf("unable to open database: %s", err)
+	}
+
+	return app{
+		db: db,
+	}, nil
+}
+
+func (a app) Get(item provider.StorageItem) (map[string]interface{}, error) {
+	var content map[string]interface{}
+
+	err := a.db.View(func(txn *badger.Txn) error {
+		entry, err := txn.Get([]byte(item.Pathname))
+		if err != nil {
+			if err == badger.ErrKeyNotFound {
+				return nil
+			}
+
+			return fmt.Errorf("unable to get entry for `%s`: %s", item.Pathname, err)
+		}
+
+		if err := entry.Value(func(val []byte) error {
+			return json.Unmarshal(val, &content)
+		}); err != nil {
+			return fmt.Errorf("unable to read entry for `%s`: %s", item.Pathname, err)
+		}
+
+		return nil
+	})
+
+	return content, err
+}
+
+func (a app) HasEntry(item provider.StorageItem) bool {
+	return a.db.View(func(txn *badger.Txn) error {
+		_, err := txn.Get([]byte(item.Pathname))
+		return err
+	}) != badger.ErrKeyNotFound
+}
+
+func (a app) Store(key, value []byte) error {
+	return a.db.Update(func(txn *badger.Txn) error {
+		return txn.Set(key, value)
+	})
+}
+
+func (a app) Close() error {
+	return a.db.Close()
+}

pkg/filesystem/fs.go

@@ -67,6 +67,10 @@ func New(config Config) (provider.Storage, error) {
 	}, nil
 }
 
+func (a app) Path(pathname string) string {
+	return path.Join(a.rootDirectory, pathname)
+}
+
 func (a *app) SetIgnoreFn(ignoreFn func(provider.StorageItem) bool) {
 	a.ignoreFn = ignoreFn
 }
@@ -77,7 +81,7 @@ func (a *app) Info(pathname string) (provider.StorageItem, error) {
 		return provider.StorageItem{}, convertError(err)
 	}
 
-	fullpath := a.getFullPath(pathname)
+	fullpath := a.Path(pathname)
 
 	info, err := os.Stat(fullpath)
 	if err != nil {
@@ -93,7 +97,7 @@ func (a *app) List(pathname string) ([]provider.StorageItem, error) {
 		return nil, convertError(err)
 	}
 
-	fullpath := a.getFullPath(pathname)
+	fullpath := a.Path(pathname)
 
 	files, err := os.ReadDir(fullpath)
 	if err != nil {
@@ -146,7 +150,7 @@ func (a *app) UpdateDate(pathname string, date time.Time) error {
 		return convertError(err)
 	}
 
-	return convertError(os.Chtimes(a.getFullPath(pathname), date, date))
+	return convertError(os.Chtimes(a.Path(pathname), date, date))
 }
 
 // Walk browses item recursively
@@ -177,7 +181,7 @@ func (a *app) CreateDir(name string) error {
 		return convertError(err)
 	}
 
-	return convertError(os.MkdirAll(a.getFullPath(name), 0700))
+	return convertError(os.MkdirAll(a.Path(name), 0700))
 }
 
 // Store file to storage
@@ -229,7 +233,7 @@ func (a *app) Rename(oldName, newName string) error {
 		return convertError(err)
 	}
 
-	return convertError(os.Rename(a.getFullPath(oldName), a.getFullPath(newName)))
+	return convertError(os.Rename(a.Path(oldName), a.Path(newName)))
 }
 
 // Remove file or directory from storage
@@ -238,5 +242,5 @@ func (a *app) Remove(pathname string) error {
 		return convertError(err)
 	}
 
-	return convertError(os.RemoveAll(a.getFullPath(pathname)))
+	return convertError(os.RemoveAll(a.Path(pathname)))
 }

pkg/filesystem/utils.go

@@ -3,7 +3,6 @@ package filesystem
 import (
 	"io"
 	"os"
-	"path"
 	"strings"
 
 	"github.com/ViBiOh/fibr/pkg/provider"
@@ -21,16 +20,12 @@ func checkPathname(pathname string) error {
 	return nil
 }
 
-func (a app) getFullPath(pathname string) string {
-	return path.Join(a.rootDirectory, pathname)
-}
-
 func (a app) getRelativePath(pathname string) string {
 	return strings.TrimPrefix(pathname, a.rootDirectory)
 }
 
 func (a app) getFile(filename string, flags int) (*os.File, error) {
-	return os.OpenFile(a.getFullPath(filename), flags, getMode(filename))
+	return os.OpenFile(a.Path(filename), flags, getMode(filename))
 }
 
 func (a app) getWritableFile(filename string) (io.WriteCloser, error) {

pkg/filesystem/utils_test.go

@@ -69,7 +69,7 @@ func TestGetFullPath(t *testing.T) {
 
 	for _, tc := range cases {
 		t.Run(tc.intention, func(t *testing.T) {
-			if got := tc.instance.getFullPath(tc.args.pathname); got != tc.want {
+			if got := tc.instance.Path(tc.args.pathname); got != tc.want {
 				t.Errorf("getFullPath() = `%s`, want `%s`", got, tc.want)
 			}
 		})

pkg/provider/provider.go

@@ -38,6 +38,7 @@ type ReadSeekerCloser interface {
 
 // Storage describe action on a storage provider
 type Storage interface {
+	Path(pathname string) string
 	SetIgnoreFn(ignoreFn func(StorageItem) bool)
 	Info(pathname string) (StorageItem, error)
 	List(pathname string) ([]StorageItem, error)

pkg/provider/providertest/storage.go

@@ -26,6 +26,11 @@ func (s stubReadCloserSeeker) Close() error {
 type Storage struct {
 }
 
+// Path fakes implementation
+func (s Storage) Path(_ string) string {
+	return ""
+}
+
 // SetIgnoreFn fakes implementation
 func (s Storage) SetIgnoreFn(func(provider.StorageItem) bool) {
 	// mock implementation