From 642811ab7b6e4ddcafb00e6c0074e7ccedf6222b Mon Sep 17 00:00:00 2001 From: Vincent Boutour Date: Tue, 27 Dec 2022 10:14:02 +0100 Subject: [PATCH] refactor(search): Extracting search to prepare virtual folder" Signed-off-by: Vincent Boutour --- cmd/fibr/config.go | 3 + cmd/fibr/fibr.go | 21 +- cmd/fibr/static/styles/main.min.css | 2 +- cmd/fibr/static/styles/search.css | 4 + pkg/crud/crud.go | 35 +-- pkg/crud/get.go | 16 +- pkg/crud/search.go | 240 ------------------ pkg/search/model.go | 110 ++++++++ pkg/search/search.go | 78 ++++++ pkg/search/util.go | 75 ++++++ .../search_test.go => search/util_test.go} | 2 +- pkg/storage/storage.go | 51 ++++ 12 files changed, 357 insertions(+), 280 deletions(-) delete mode 100644 pkg/crud/search.go create mode 100644 pkg/search/model.go create mode 100644 pkg/search/search.go create mode 100644 pkg/search/util.go rename pkg/{crud/search_test.go => search/util_test.go} (98%) create mode 100644 pkg/storage/storage.go diff --git a/cmd/fibr/config.go b/cmd/fibr/config.go index e444c23b..73513b96 100644 --- a/cmd/fibr/config.go +++ b/cmd/fibr/config.go @@ -10,6 +10,7 @@ import ( "github.com/ViBiOh/fibr/pkg/crud" "github.com/ViBiOh/fibr/pkg/exif" "github.com/ViBiOh/fibr/pkg/share" + "github.com/ViBiOh/fibr/pkg/storage" "github.com/ViBiOh/fibr/pkg/thumbnail" "github.com/ViBiOh/fibr/pkg/webhook" "github.com/ViBiOh/flags" @@ -36,6 +37,7 @@ type configuration struct { health health.Config owasp owasp.Config basic basicMemory.Config + storage storage.Config crud crud.Config share share.Config webhook webhook.Config @@ -66,6 +68,7 @@ func newConfig() (configuration, error) { prometheus: prometheus.Flags(fs, "prometheus", flags.NewOverride("Gzip", false)), owasp: owasp.Flags(fs, "", flags.NewOverride("FrameOptions", "SAMEORIGIN"), flags.NewOverride("Csp", "default-src 'self'; base-uri 'self'; script-src 'self' 'httputils-nonce' unpkg.com/webp-hero@0.0.2/dist-cjs/ unpkg.com/leaflet@1.9.3/dist/ unpkg.com/leaflet.markercluster@1.5.1/; style-src 'self' 'httputils-nonce' unpkg.com/leaflet@1.9.3/dist/ unpkg.com/leaflet.markercluster@1.5.1/; img-src 'self' data: a.tile.openstreetmap.org b.tile.openstreetmap.org c.tile.openstreetmap.org")), basic: basicMemory.Flags(fs, "auth", flags.NewOverride("Profiles", "1:admin")), + storage: storage.Flags(fs, ""), crud: crud.Flags(fs, ""), share: share.Flags(fs, "share"), webhook: webhook.Flags(fs, "webhook"), diff --git a/cmd/fibr/fibr.go b/cmd/fibr/fibr.go index 73e1fce9..fe9d0594 100644 --- a/cmd/fibr/fibr.go +++ b/cmd/fibr/fibr.go @@ -17,7 +17,9 @@ import ( "github.com/ViBiOh/fibr/pkg/exif" "github.com/ViBiOh/fibr/pkg/fibr" "github.com/ViBiOh/fibr/pkg/provider" + "github.com/ViBiOh/fibr/pkg/search" "github.com/ViBiOh/fibr/pkg/share" + "github.com/ViBiOh/fibr/pkg/storage" "github.com/ViBiOh/fibr/pkg/thumbnail" "github.com/ViBiOh/fibr/pkg/webhook" "github.com/ViBiOh/httputils/v4/pkg/alcotest" @@ -76,25 +78,28 @@ func main() { prometheusRegisterer := client.prometheus.Registerer() - storageProvider, err := absto.New(config.absto, client.tracer.GetTracer("storage")) + storageApp, err := absto.New(config.absto, client.tracer.GetTracer("storage")) + logger.Fatal(err) + + filteredStorage, err := storage.Get(config.storage, storageApp) logger.Fatal(err) eventBus, err := provider.NewEventBus(provider.MaxConcurrency, prometheusRegisterer, client.tracer.GetTracer("bus")) logger.Fatal(err) - thumbnailApp, err := thumbnail.New(config.thumbnail, storageProvider, client.redis, prometheusRegisterer, client.tracer, client.amqp) + thumbnailApp, err := thumbnail.New(config.thumbnail, storageApp, client.redis, prometheusRegisterer, client.tracer, client.amqp) logger.Fatal(err) rendererApp, err := renderer.New(config.renderer, content, fibr.FuncMap, client.tracer.GetTracer("renderer")) logger.Fatal(err) - exifApp, err := exif.New(config.exif, storageProvider, prometheusRegisterer, client.tracer, client.amqp, client.redis) + exifApp, err := exif.New(config.exif, storageApp, prometheusRegisterer, client.tracer, client.amqp, client.redis) logger.Fatal(err) - webhookApp, err := webhook.New(config.webhook, storageProvider, prometheusRegisterer, client.amqp, rendererApp, thumbnailApp) + webhookApp, err := webhook.New(config.webhook, storageApp, prometheusRegisterer, client.amqp, rendererApp, thumbnailApp) logger.Fatal(err) - shareApp, err := share.New(config.share, storageProvider, client.amqp) + shareApp, err := share.New(config.share, storageApp, client.amqp) logger.Fatal(err) amqpThumbnailApp, err := amqphandler.New(config.amqpThumbnail, client.amqp, client.tracer.GetTracer("amqp_handler_thumbnail"), thumbnailApp.AMQPHandler) @@ -109,7 +114,9 @@ func main() { amqpWebhookApp, err := amqphandler.New(config.amqpWebhook, client.amqp, client.tracer.GetTracer("amqp_handler_webhook"), webhookApp.AMQPHandler) logger.Fatal(err) - crudApp, err := crud.New(config.crud, storageProvider, rendererApp, shareApp, webhookApp, thumbnailApp, exifApp, eventBus.Push, client.amqp, client.tracer.GetTracer("crud")) + searchApp := search.New(filteredStorage, thumbnailApp, exifApp, client.tracer.GetTracer("search")) + + crudApp, err := crud.New(config.crud, storageApp, filteredStorage, rendererApp, shareApp, webhookApp, thumbnailApp, exifApp, searchApp, eventBus.Push, client.amqp, client.tracer.GetTracer("crud")) logger.Fatal(err) var middlewareApp provider.Auth @@ -130,7 +137,7 @@ func main() { go webhookApp.Start(ctx) go shareApp.Start(ctx) go crudApp.Start(ctx) - go eventBus.Start(ctx, storageProvider, []provider.Renamer{thumbnailApp.Rename, exifApp.Rename}, shareApp.EventConsumer, thumbnailApp.EventConsumer, exifApp.EventConsumer, webhookApp.EventConsumer) + go eventBus.Start(ctx, storageApp, []provider.Renamer{thumbnailApp.Rename, exifApp.Rename}, shareApp.EventConsumer, thumbnailApp.EventConsumer, exifApp.EventConsumer, webhookApp.EventConsumer) go promServer.Start("prometheus", client.health.End(), client.prometheus.Handler()) go appServer.Start("http", client.health.End(), httputils.Handler(handler, client.health, recoverer.Middleware, client.prometheus.Middleware, client.tracer.Middleware, owasp.New(config.owasp).Middleware)) diff --git a/cmd/fibr/static/styles/main.min.css b/cmd/fibr/static/styles/main.min.css index 3c34b3e4..950b2fd6 100644 --- a/cmd/fibr/static/styles/main.min.css +++ b/cmd/fibr/static/styles/main.min.css @@ -1 +1 @@ -.thumbnail{max-height:100%;vertical-align:middle;width:100%}.exif-button{bottom:1rem;position:absolute;right:1rem}#exif-modal:target{display:flex;z-index:5}#exif-modal:target~.content{pointer-events:none}.breakable{word-break:break-word}:root{--primary:royalblue;--success:limegreen;--danger:crimson;--dark:#272727;--grey:#3b3b3b;--white:aliceblue;--icon-size:2.4rem;--icon-large:4.8rem}*{box-sizing:border-box}html{font-size:62.5%}body{-webkit-overflow-scrolling:touch;background-color:var(--dark);height:100vh}body,button,input{color:var(--white);font-family:-apple-system,segoe ui,roboto,oxygen-sans,ubuntu,cantarell,helvetica nue,sans-serif;font-size:1.6rem;font-style:normal;font-weight:400}input{color:var(--dark)}input[type=file]{color:var(--white)}a{color:var(--white);text-decoration:none}a:hover{color:var(--primary);text-decoration:underline}.primary{color:var(--primary)}.success{color:var(--success)}.danger{color:var(--danger)}.grey{color:var(--grey)}.white{color:var(--white)}.bg-primary,.bg-primary:hover{background-color:var(--primary);color:var(--white);text-decoration:none}.bg-success,.bg-success:hover{background-color:var(--success);color:var(--dark);text-decoration:none}.bg-danger,.bg-danger:hover{background-color:var(--danger);color:var(--white);text-decoration:none}.bg-grey,.bg-grey:hover{background-color:var(--grey);color:var(--white);text-decoration:none}.button{border-radius:4px;border:0;cursor:pointer;display:inline-block;margin:0;padding:1rem;text-decoration:none}.button-icon{background-color:initial}.icon{background-position:50%;background-repeat:no-repeat;color:var(--white);display:inline-block;height:var(--icon-size);text-decoration:none;vertical-align:middle}.icon-square{width:var(--icon-size)}.icon-bottom{vertical-align:bottom}.icon-large{height:var(--icon-large);width:var(--icon-large)}.icon-overlay{height:var(--icon-large);left:calc((100% - var(--icon-large))/2);pointer-events:none;position:absolute;top:calc((100% - var(--icon-large))/2);width:var(--icon-large)}.icon-overlay-small{bottom:1rem;height:var(--icon-size);position:absolute;right:1rem;width:var(--icon-size)}.clickable{cursor:pointer}.modal{align-items:flex-start;background-color:rgba(84,84,84,.75);display:none;height:100vh;justify-content:center;left:0;padding-top:5rem;pointer-events:none;position:fixed;top:0;width:100vw}.modal-content{background-color:var(--dark);display:flex;flex-direction:column;max-height:80%;max-width:90%;pointer-events:auto}.header{background-color:var(--grey);margin-top:0;padding:.5rem 1rem;text-align:left}.center{text-align:center}.padding{padding:1rem}.no-padding{padding:0}.margin{margin:1rem}.margin-left{margin-left:1rem}.no-margin{margin:0}.no-background{background-color:initial}.hidden{display:none}@media print{body::before{content:'Save ink, share link.'}body>*{display:none}}.flex{display:flex}.flex-center{align-items:center;justify-content:center}.flex-grow{flex:1 1}.flex-column{flex-direction:column}.full{width:100%}.full-screen{max-width:100vw}.medium{font-size:2.4rem}.small{font-size:1.6rem}.scrollable{overflow-y:auto}.padding-left{padding-left:1rem}.padding-right{padding-right:1rem}.ellipsis{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.flex-ellipsis{min-width:0}.relative{position:relative}.block{display:block}@media screen and (max-width:374px){.hide-xs{display:none}}@media screen and (max-width:599px){.hide-s{display:none}}#map:target{display:flex;z-index:5}#map:target~.content{pointer-events:none}#map-container{background-color:var(--grey);color:var(--dark);height:80vh;width:80vw}.thumbnail-img{height:15rem}.thumbnail-popup{bottom:55px!important;height:15rem;width:19rem}.map-throbber{z-index:500}.map-throbber .throbber-dot{height:3.2rem;width:3.2rem}#search:target{display:flex;z-index:5}#search:target~.content{pointer-events:none}#size{width:7rem}.share-form-modal:target{display:flex;z-index:5}.share-form-modal:target~.content{pointer-events:none}.share-list-modal:target{display:flex;z-index:5}.share-list-modal:target~.content{pointer-events:none}#shares{border-spacing:0;display:block;overflow-x:hidden;overflow-y:auto}#shares th,#shares td{padding:1rem}.share-link{color:var(--success)}.share-content:hover{background-color:var(--grey)}.path{max-width:30rem;text-align:left}@media screen and (max-width:485px){#shares th,#shares td{padding:.5rem}.path{max-width:20rem}}@media screen and (max-width:430px){.path{max-width:16rem}}@media screen and (max-width:375px){.path{max-width:12rem}}@media screen and (max-width:320px){.path{max-width:8rem}}.stats{margin:0 auto;width:30rem}.key{display:inline-block;width:10rem}.throbber{margin:auto;text-align:center}.throbber-dot{animation:throbber-dot 1.4s infinite ease-in-out both;background-color:var(--dark);border-radius:100%;display:inline-block;height:1rem;width:1rem}.throbber-white .throbber-dot{background-color:var(--white)}.throbber-dot-1{animation-delay:-.32s}.throbber-dot-2{animation-delay:-.16s}@keyframes throbber-dot{0%,80%,100%{transform:scale(0)}40%{transform:scale(1)}}.upload-modal:target,.upload-success:target{display:flex;z-index:5}.upload-modal:target~.content,.upload-success:target~.content{pointer-events:none}.upload-width{max-width:30rem}.upload-item{display:inline-block;width:calc(100% - 2rem)}.upload-name{text-align:left}.opacity{flex:0 0;opacity:0}.upload-status{margin-left:.5rem;text-align:right;width:2rem}#webhook-form:target{display:flex;z-index:5}#webhook-form:target~.content{pointer-events:none}#webhook-list:target{display:flex;z-index:5}#webhook-list:target~.content{pointer-events:none}#webhooks{border-spacing:0;display:block;overflow:auto}#webhooks th,#webhooks td{padding:1rem}.webhook-content:hover{background-color:var(--grey)}.url{max-width:30rem;text-align:left}@media screen and (max-width:485px){.url{max-width:20rem}}@media screen and (max-width:430px){.url{max-width:16rem}}@media screen and (max-width:375px){.url{max-width:12rem}}@media screen and (max-width:320px){.url{max-width:8rem}} \ No newline at end of file +.thumbnail{max-height:100%;vertical-align:middle;width:100%}.exif-button{bottom:1rem;position:absolute;right:1rem}#exif-modal:target{display:flex;z-index:5}#exif-modal:target~.content{pointer-events:none}.breakable{word-break:break-word}:root{--primary:royalblue;--success:limegreen;--danger:crimson;--dark:#272727;--grey:#3b3b3b;--white:aliceblue;--icon-size:2.4rem;--icon-large:4.8rem}*{box-sizing:border-box}html{font-size:62.5%}body{-webkit-overflow-scrolling:touch;background-color:var(--dark);height:100vh}body,button,input{color:var(--white);font-family:-apple-system,segoe ui,roboto,oxygen-sans,ubuntu,cantarell,helvetica nue,sans-serif;font-size:1.6rem;font-style:normal;font-weight:400}input{color:var(--dark)}input[type=file]{color:var(--white)}a{color:var(--white);text-decoration:none}a:hover{color:var(--primary);text-decoration:underline}.primary{color:var(--primary)}.success{color:var(--success)}.danger{color:var(--danger)}.grey{color:var(--grey)}.white{color:var(--white)}.bg-primary,.bg-primary:hover{background-color:var(--primary);color:var(--white);text-decoration:none}.bg-success,.bg-success:hover{background-color:var(--success);color:var(--dark);text-decoration:none}.bg-danger,.bg-danger:hover{background-color:var(--danger);color:var(--white);text-decoration:none}.bg-grey,.bg-grey:hover{background-color:var(--grey);color:var(--white);text-decoration:none}.button{border-radius:4px;border:0;cursor:pointer;display:inline-block;margin:0;padding:1rem;text-decoration:none}.button-icon{background-color:initial}.icon{background-position:50%;background-repeat:no-repeat;color:var(--white);display:inline-block;height:var(--icon-size);text-decoration:none;vertical-align:middle}.icon-square{width:var(--icon-size)}.icon-bottom{vertical-align:bottom}.icon-large{height:var(--icon-large);width:var(--icon-large)}.icon-overlay{height:var(--icon-large);left:calc((100% - var(--icon-large))/2);pointer-events:none;position:absolute;top:calc((100% - var(--icon-large))/2);width:var(--icon-large)}.icon-overlay-small{bottom:1rem;height:var(--icon-size);position:absolute;right:1rem;width:var(--icon-size)}.clickable{cursor:pointer}.modal{align-items:flex-start;background-color:rgba(84,84,84,.75);display:none;height:100vh;justify-content:center;left:0;padding-top:5rem;pointer-events:none;position:fixed;top:0;width:100vw}.modal-content{background-color:var(--dark);display:flex;flex-direction:column;max-height:80%;max-width:90%;pointer-events:auto}.header{background-color:var(--grey);margin-top:0;padding:.5rem 1rem;text-align:left}.center{text-align:center}.padding{padding:1rem}.no-padding{padding:0}.margin{margin:1rem}.margin-left{margin-left:1rem}.no-margin{margin:0}.no-background{background-color:initial}.hidden{display:none}@media print{body::before{content:'Save ink, share link.'}body>*{display:none}}.flex{display:flex}.flex-center{align-items:center;justify-content:center}.flex-grow{flex:1 1}.flex-column{flex-direction:column}.full{width:100%}.full-screen{max-width:100vw}.medium{font-size:2.4rem}.small{font-size:1.6rem}.scrollable{overflow-y:auto}.padding-left{padding-left:1rem}.padding-right{padding-right:1rem}.ellipsis{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.flex-ellipsis{min-width:0}.relative{position:relative}.block{display:block}@media screen and (max-width:374px){.hide-xs{display:none}}@media screen and (max-width:599px){.hide-s{display:none}}#map:target{display:flex;z-index:5}#map:target~.content{pointer-events:none}#map-container{background-color:var(--grey);color:var(--dark);height:80vh;width:80vw}.thumbnail-img{height:15rem}.thumbnail-popup{bottom:55px!important;height:15rem;width:19rem}.map-throbber{z-index:500}.map-throbber .throbber-dot{height:3.2rem;width:3.2rem}#search:target{display:flex;z-index:5}#search:target~.content{pointer-events:none}#types{height:14.1rem}#size{width:7rem}.share-form-modal:target{display:flex;z-index:5}.share-form-modal:target~.content{pointer-events:none}.share-list-modal:target{display:flex;z-index:5}.share-list-modal:target~.content{pointer-events:none}#shares{border-spacing:0;display:block;overflow-x:hidden;overflow-y:auto}#shares th,#shares td{padding:1rem}.share-link{color:var(--success)}.share-content:hover{background-color:var(--grey)}.path{max-width:30rem;text-align:left}@media screen and (max-width:485px){#shares th,#shares td{padding:.5rem}.path{max-width:20rem}}@media screen and (max-width:430px){.path{max-width:16rem}}@media screen and (max-width:375px){.path{max-width:12rem}}@media screen and (max-width:320px){.path{max-width:8rem}}.stats{margin:0 auto;width:30rem}.key{display:inline-block;width:10rem}.throbber{margin:auto;text-align:center}.throbber-dot{animation:throbber-dot 1.4s infinite ease-in-out both;background-color:var(--dark);border-radius:100%;display:inline-block;height:1rem;width:1rem}.throbber-white .throbber-dot{background-color:var(--white)}.throbber-dot-1{animation-delay:-.32s}.throbber-dot-2{animation-delay:-.16s}@keyframes throbber-dot{0%,80%,100%{transform:scale(0)}40%{transform:scale(1)}}.upload-modal:target,.upload-success:target{display:flex;z-index:5}.upload-modal:target~.content,.upload-success:target~.content{pointer-events:none}.upload-width{max-width:30rem}.upload-item{display:inline-block;width:calc(100% - 2rem)}.upload-name{text-align:left}.opacity{flex:0 0;opacity:0}.upload-status{margin-left:.5rem;text-align:right;width:2rem}#webhook-form:target{display:flex;z-index:5}#webhook-form:target~.content{pointer-events:none}#webhook-list:target{display:flex;z-index:5}#webhook-list:target~.content{pointer-events:none}#webhooks{border-spacing:0;display:block;overflow:auto}#webhooks th,#webhooks td{padding:1rem}.webhook-content:hover{background-color:var(--grey)}.url{max-width:30rem;text-align:left}@media screen and (max-width:485px){.url{max-width:20rem}}@media screen and (max-width:430px){.url{max-width:16rem}}@media screen and (max-width:375px){.url{max-width:12rem}}@media screen and (max-width:320px){.url{max-width:8rem}} \ No newline at end of file diff --git a/cmd/fibr/static/styles/search.css b/cmd/fibr/static/styles/search.css index 72ddbe84..c288e433 100644 --- a/cmd/fibr/static/styles/search.css +++ b/cmd/fibr/static/styles/search.css @@ -7,6 +7,10 @@ pointer-events: none; } +#types { + height: 14.1rem; +} + #size { width: 7rem; } diff --git a/pkg/crud/crud.go b/pkg/crud/crud.go index 70dbdd95..7fe03c36 100644 --- a/pkg/crud/crud.go +++ b/pkg/crud/crud.go @@ -6,13 +6,13 @@ import ( "flag" "fmt" "net/http" - "regexp" "strings" "time" absto "github.com/ViBiOh/absto/pkg/model" "github.com/ViBiOh/fibr/pkg/exif" "github.com/ViBiOh/fibr/pkg/provider" + "github.com/ViBiOh/fibr/pkg/search" "github.com/ViBiOh/fibr/pkg/thumbnail" "github.com/ViBiOh/flags" "github.com/ViBiOh/httputils/v4/pkg/amqp" @@ -36,6 +36,7 @@ type App struct { shareApp provider.ShareManager webhookApp provider.WebhookManager exifApp provider.ExifManager + searchApp search.App pushEvent provider.EventProducer amqpClient *amqp.Client @@ -50,7 +51,6 @@ type App struct { } type Config struct { - ignore *string amqpExclusiveRoutingKey *string bcryptDuration *string temporaryFolder *string @@ -60,7 +60,6 @@ type Config struct { func Flags(fs *flag.FlagSet, prefix string) Config { return Config{ - ignore: flags.String(fs, prefix, "crud", "IgnorePattern", "Ignore pattern when listing files or directory", "", nil), sanitizeOnStart: flags.Bool(fs, prefix, "crud", "SanitizeOnStart", "Sanitize name on start", false, nil), bcryptDuration: flags.String(fs, prefix, "crud", "BcryptDuration", "Wanted bcrypt duration for calculating effective cost", "0.25s", nil), @@ -71,7 +70,7 @@ func Flags(fs *flag.FlagSet, prefix string) Config { } } -func New(config Config, storage absto.Storage, rendererApp renderer.App, shareApp provider.ShareManager, webhookApp provider.WebhookManager, thumbnailApp thumbnail.App, exifApp exif.App, eventProducer provider.EventProducer, amqpClient *amqp.Client, tracer trace.Tracer) (App, error) { +func New(config Config, storageApp absto.Storage, filteredStorage absto.Storage, rendererApp renderer.App, shareApp provider.ShareManager, webhookApp provider.WebhookManager, thumbnailApp thumbnail.App, exifApp exif.App, searchApp search.App, eventProducer provider.EventProducer, amqpClient *amqp.Client, tracer trace.Tracer) (App, error) { app := App{ sanitizeOnStart: *config.sanitizeOnStart, @@ -81,12 +80,14 @@ func New(config Config, storage absto.Storage, rendererApp renderer.App, shareAp tracer: tracer, pushEvent: eventProducer, - rawStorageApp: storage, + rawStorageApp: storageApp, + storageApp: filteredStorage, rendererApp: rendererApp, thumbnailApp: thumbnailApp, exifApp: exifApp, shareApp: shareApp, webhookApp: webhookApp, + searchApp: searchApp, amqpClient: amqpClient, amqpExclusiveRoutingKey: strings.TrimSpace(*config.amqpExclusiveRoutingKey), @@ -98,30 +99,6 @@ func New(config Config, storage absto.Storage, rendererApp renderer.App, shareAp } } - var ignorePattern *regexp.Regexp - ignore := *config.ignore - if len(ignore) != 0 { - pattern, err := regexp.Compile(ignore) - if err != nil { - return App{}, err - } - - ignorePattern = pattern - logger.Info("Ignoring files with pattern `%s`", ignore) - } - - app.storageApp = storage.WithIgnoreFn(func(item absto.Item) bool { - if strings.HasPrefix(item.Pathname, provider.MetadataDirectoryName) { - return true - } - - if ignorePattern != nil && ignorePattern.MatchString(item.Name) { - return true - } - - return false - }) - bcryptDuration, err := time.ParseDuration(strings.TrimSpace(*config.bcryptDuration)) if err != nil { return app, fmt.Errorf("parse bcrypt duration: %w", err) diff --git a/pkg/crud/get.go b/pkg/crud/get.go index 26f67923..59502bd7 100644 --- a/pkg/crud/get.go +++ b/pkg/crud/get.go @@ -133,7 +133,19 @@ func (a App) handleDir(w http.ResponseWriter, r *http.Request, request provider. go a.notify(tracer.CopyToBackground(r.Context()), provider.NewAccessEvent(item, r)) if query.GetBool(r, "search") { - return a.search(r, request, items) + files, hasMap, err := a.searchApp.Search(r, request, items) + if err != nil { + return errorReturn(request, err) + } + + return renderer.NewPage("search", http.StatusOK, map[string]any{ + "Paths": getPathParts(request), + "Files": files, + "Cover": a.getCover(r.Context(), request, items), + "Search": r.URL.Query(), + "Request": request, + "HasMap": hasMap, + }), nil } provider.SetPrefsCookie(w, request) @@ -150,7 +162,7 @@ func (a App) listFiles(r *http.Request, request provider.Request, item absto.Ite defer end() if query.GetBool(r, "search") { - items, err = a.searchFiles(r, request) + items, err = a.searchApp.Files(r, request) } else { items, err = a.storageApp.List(ctx, request.Filepath()) } diff --git a/pkg/crud/search.go b/pkg/crud/search.go deleted file mode 100644 index a1e6cddf..00000000 --- a/pkg/crud/search.go +++ /dev/null @@ -1,240 +0,0 @@ -package crud - -import ( - "fmt" - "net/http" - "net/url" - "regexp" - "strconv" - "strings" - "time" - - absto "github.com/ViBiOh/absto/pkg/model" - "github.com/ViBiOh/fibr/pkg/provider" - "github.com/ViBiOh/fibr/pkg/thumbnail" - httpModel "github.com/ViBiOh/httputils/v4/pkg/model" - "github.com/ViBiOh/httputils/v4/pkg/renderer" - "github.com/ViBiOh/httputils/v4/pkg/tracer" -) - -const ( - isoDateLayout = "2006-01-02" - kilobytes = 1 << 10 - megabytes = 1 << 20 - gigabytes = 1 << 30 -) - -type search struct { - pattern *regexp.Regexp - before time.Time - after time.Time - mimes []string - size int64 - greaterThan bool -} - -func parseSearch(params url.Values) (output search, err error) { - if name := strings.TrimSpace(params.Get("name")); len(name) > 0 { - output.pattern, err = regexp.Compile(name) - if err != nil { - return - } - } - - output.before, err = parseDate(strings.TrimSpace(params.Get("before"))) - if err != nil { - return - } - - output.after, err = parseDate(strings.TrimSpace(params.Get("after"))) - if err != nil { - return - } - - rawSize := strings.TrimSpace(params.Get("size")) - if len(rawSize) > 0 { - output.size, err = strconv.ParseInt(rawSize, 10, 64) - if err != nil { - return - } - } - - output.size = computeSize(strings.TrimSpace(params.Get("sizeUnit")), output.size) - output.greaterThan = strings.TrimSpace(params.Get("sizeOrder")) == "gt" - output.mimes = computeMimes(params["types"]) - - return -} - -func (s search) match(item absto.Item) bool { - if !s.matchSize(item) { - return false - } - - if !s.before.IsZero() && item.Date.After(s.before) { - return false - } - - if !s.after.IsZero() && item.Date.Before(s.after) { - return false - } - - if !s.matchMimes(item) { - return false - } - - if s.pattern != nil && !s.pattern.MatchString(item.Pathname) { - return false - } - - return true -} - -func (s search) matchSize(item absto.Item) bool { - if s.size == 0 { - return true - } - - if (s.size - item.Size) > 0 == s.greaterThan { - return false - } - - return true -} - -func (s search) matchMimes(item absto.Item) bool { - if len(s.mimes) == 0 { - return true - } - - for _, mime := range s.mimes { - if strings.EqualFold(mime, item.Extension) { - return true - } - } - - return false -} - -func (a App) searchFiles(r *http.Request, request provider.Request) (items []absto.Item, err error) { - params := r.URL.Query() - - criterions, err := parseSearch(params) - if err != nil { - return nil, httpModel.WrapInvalid(err) - } - - err = a.storageApp.Walk(r.Context(), request.Filepath(), func(item absto.Item) error { - if item.IsDir || !criterions.match(item) { - return nil - } - - items = append(items, item) - - return nil - }) - - return -} - -func (a App) search(r *http.Request, request provider.Request, files []absto.Item) (renderer.Page, error) { - ctx, end := tracer.StartSpan(r.Context(), a.tracer, "search") - defer end() - - items := make([]provider.RenderItem, len(files)) - var hasMap bool - - renderWithThumbnail := request.Display == provider.GridDisplay - - for i, item := range files { - renderItem := provider.StorageToRender(item, request) - - if renderWithThumbnail && a.thumbnailApp.CanHaveThumbnail(item) && a.thumbnailApp.HasThumbnail(ctx, item, thumbnail.SmallSize) { - renderItem.HasThumbnail = true - } - - items[i] = renderItem - - if !hasMap { - if exif, err := a.exifApp.GetExifFor(ctx, item); err == nil && exif.Geocode.Longitude != 0 && exif.Geocode.Latitude != 0 { - hasMap = true - } - } - } - - return renderer.NewPage("search", http.StatusOK, map[string]any{ - "Paths": getPathParts(request), - "Files": items, - "Cover": a.getCover(ctx, request, files), - "Search": r.URL.Query(), - "Request": request, - "HasMap": hasMap, - }), nil -} - -func computeSize(unit string, size int64) int64 { - switch unit { - case "kb": - return kilobytes * size - case "mb": - return megabytes * size - case "gb": - return gigabytes * size - default: - return size - } -} - -func computeMimes(aliases []string) []string { - var output []string - - for _, alias := range aliases { - switch alias { - case "archive": - return append(output, getKeysOfMap(provider.ArchiveExtensions)...) - case "audio": - return append(output, getKeysOfMap(provider.AudioExtensions)...) - case "code": - return append(output, getKeysOfMap(provider.CodeExtensions)...) - case "excel": - return append(output, getKeysOfMap(provider.ExcelExtensions)...) - case "image": - return append(output, getKeysOfMap(provider.ImageExtensions)...) - case "pdf": - return append(output, getKeysOfMap(provider.PdfExtensions)...) - case "video": - return append(output, getKeysOfMap(provider.VideoExtensions)...) - case "stream": - return append(output, getKeysOfMap(provider.StreamExtensions)...) - case "word": - return append(output, getKeysOfMap(provider.WordExtensions)...) - } - } - - return output -} - -func getKeysOfMap[T any](input map[string]T) []string { - output := make([]string, len(input)) - var i int64 - - for key := range input { - output[i] = key - i++ - } - - return output -} - -func parseDate(raw string) (time.Time, error) { - if len(raw) == 0 { - return time.Time{}, nil - } - - value, err := time.Parse(isoDateLayout, raw) - if err != nil { - return time.Time{}, fmt.Errorf("parse date: %w", err) - } - - return value, nil -} diff --git a/pkg/search/model.go b/pkg/search/model.go new file mode 100644 index 00000000..8689c9a4 --- /dev/null +++ b/pkg/search/model.go @@ -0,0 +1,110 @@ +package search + +import ( + "net/url" + "regexp" + "strconv" + "strings" + "time" + + absto "github.com/ViBiOh/absto/pkg/model" +) + +const ( + isoDateLayout = "2006-01-02" + kilobytes = 1 << 10 + megabytes = 1 << 20 + gigabytes = 1 << 30 +) + +type search struct { + pattern *regexp.Regexp + before time.Time + after time.Time + mimes []string + size int64 + greaterThan bool +} + +func parseSearch(params url.Values) (output search, err error) { + if name := strings.TrimSpace(params.Get("name")); len(name) > 0 { + output.pattern, err = regexp.Compile(name) + if err != nil { + return + } + } + + output.before, err = parseDate(strings.TrimSpace(params.Get("before"))) + if err != nil { + return + } + + output.after, err = parseDate(strings.TrimSpace(params.Get("after"))) + if err != nil { + return + } + + rawSize := strings.TrimSpace(params.Get("size")) + if len(rawSize) > 0 { + output.size, err = strconv.ParseInt(rawSize, 10, 64) + if err != nil { + return + } + } + + output.size = computeSize(strings.TrimSpace(params.Get("sizeUnit")), output.size) + output.greaterThan = strings.TrimSpace(params.Get("sizeOrder")) == "gt" + output.mimes = computeMimes(params["types"]) + + return +} + +func (s search) match(item absto.Item) bool { + if !s.matchSize(item) { + return false + } + + if !s.before.IsZero() && item.Date.After(s.before) { + return false + } + + if !s.after.IsZero() && item.Date.Before(s.after) { + return false + } + + if !s.matchMimes(item) { + return false + } + + if s.pattern != nil && !s.pattern.MatchString(item.Pathname) { + return false + } + + return true +} + +func (s search) matchSize(item absto.Item) bool { + if s.size == 0 { + return true + } + + if (s.size - item.Size) > 0 == s.greaterThan { + return false + } + + return true +} + +func (s search) matchMimes(item absto.Item) bool { + if len(s.mimes) == 0 { + return true + } + + for _, mime := range s.mimes { + if strings.EqualFold(mime, item.Extension) { + return true + } + } + + return false +} diff --git a/pkg/search/search.go b/pkg/search/search.go new file mode 100644 index 00000000..df8e7e20 --- /dev/null +++ b/pkg/search/search.go @@ -0,0 +1,78 @@ +package search + +import ( + "net/http" + + absto "github.com/ViBiOh/absto/pkg/model" + "github.com/ViBiOh/fibr/pkg/exif" + "github.com/ViBiOh/fibr/pkg/provider" + "github.com/ViBiOh/fibr/pkg/thumbnail" + httpModel "github.com/ViBiOh/httputils/v4/pkg/model" + "github.com/ViBiOh/httputils/v4/pkg/tracer" + "go.opentelemetry.io/otel/trace" +) + +type App struct { + tracer trace.Tracer + storageApp absto.Storage + exifApp provider.ExifManager + thumbnailApp thumbnail.App +} + +func New(storageApp absto.Storage, thumbnailApp thumbnail.App, exifApp exif.App, tracer trace.Tracer) App { + return App{ + tracer: tracer, + storageApp: storageApp, + thumbnailApp: thumbnailApp, + exifApp: exifApp, + } +} + +func (a App) Files(r *http.Request, request provider.Request) (items []absto.Item, err error) { + params := r.URL.Query() + + criterions, err := parseSearch(params) + if err != nil { + return nil, httpModel.WrapInvalid(err) + } + + err = a.storageApp.Walk(r.Context(), request.Filepath(), func(item absto.Item) error { + if item.IsDir || !criterions.match(item) { + return nil + } + + items = append(items, item) + + return nil + }) + + return +} + +func (a App) Search(r *http.Request, request provider.Request, files []absto.Item) ([]provider.RenderItem, bool, error) { + ctx, end := tracer.StartSpan(r.Context(), a.tracer, "search") + defer end() + + items := make([]provider.RenderItem, len(files)) + var hasMap bool + + renderWithThumbnail := request.Display == provider.GridDisplay + + for i, item := range files { + renderItem := provider.StorageToRender(item, request) + + if renderWithThumbnail && a.thumbnailApp.CanHaveThumbnail(item) && a.thumbnailApp.HasThumbnail(ctx, item, thumbnail.SmallSize) { + renderItem.HasThumbnail = true + } + + items[i] = renderItem + + if !hasMap { + if exif, err := a.exifApp.GetExifFor(ctx, item); err == nil && exif.Geocode.Longitude != 0 && exif.Geocode.Latitude != 0 { + hasMap = true + } + } + } + + return items, hasMap, nil +} diff --git a/pkg/search/util.go b/pkg/search/util.go new file mode 100644 index 00000000..c2bbd29c --- /dev/null +++ b/pkg/search/util.go @@ -0,0 +1,75 @@ +package search + +import ( + "fmt" + "time" + + "github.com/ViBiOh/fibr/pkg/provider" +) + +func computeSize(unit string, size int64) int64 { + switch unit { + case "kb": + return kilobytes * size + case "mb": + return megabytes * size + case "gb": + return gigabytes * size + default: + return size + } +} + +func computeMimes(aliases []string) []string { + var output []string + + for _, alias := range aliases { + switch alias { + case "archive": + return append(output, getKeysOfMap(provider.ArchiveExtensions)...) + case "audio": + return append(output, getKeysOfMap(provider.AudioExtensions)...) + case "code": + return append(output, getKeysOfMap(provider.CodeExtensions)...) + case "excel": + return append(output, getKeysOfMap(provider.ExcelExtensions)...) + case "image": + return append(output, getKeysOfMap(provider.ImageExtensions)...) + case "pdf": + return append(output, getKeysOfMap(provider.PdfExtensions)...) + case "video": + return append(output, getKeysOfMap(provider.VideoExtensions)...) + case "stream": + return append(output, getKeysOfMap(provider.StreamExtensions)...) + case "word": + return append(output, getKeysOfMap(provider.WordExtensions)...) + } + } + + return output +} + +func getKeysOfMap[T any](input map[string]T) []string { + output := make([]string, len(input)) + var i int64 + + for key := range input { + output[i] = key + i++ + } + + return output +} + +func parseDate(raw string) (time.Time, error) { + if len(raw) == 0 { + return time.Time{}, nil + } + + value, err := time.Parse(isoDateLayout, raw) + if err != nil { + return time.Time{}, fmt.Errorf("parse date: %w", err) + } + + return value, nil +} diff --git a/pkg/crud/search_test.go b/pkg/search/util_test.go similarity index 98% rename from pkg/crud/search_test.go rename to pkg/search/util_test.go index 5da654c5..c0ae55bd 100644 --- a/pkg/crud/search_test.go +++ b/pkg/search/util_test.go @@ -1,4 +1,4 @@ -package crud +package search import ( "testing" diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go new file mode 100644 index 00000000..3df7f8a7 --- /dev/null +++ b/pkg/storage/storage.go @@ -0,0 +1,51 @@ +package storage + +import ( + "flag" + "fmt" + "regexp" + "strings" + + absto "github.com/ViBiOh/absto/pkg/model" + "github.com/ViBiOh/fibr/pkg/provider" + "github.com/ViBiOh/flags" + "github.com/ViBiOh/httputils/v4/pkg/logger" +) + +type Config struct { + ignore *string +} + +func Flags(fs *flag.FlagSet, prefix string) Config { + return Config{ + ignore: flags.String(fs, prefix, "crud", "IgnorePattern", "Ignore pattern when listing files or directory", "", nil), + } +} + +func Get(config Config, storage absto.Storage) (absto.Storage, error) { + ignore := *config.ignore + if len(ignore) == 0 { + return storage, nil + } + + pattern, err := regexp.Compile(ignore) + if err != nil { + return storage, fmt.Errorf("regexp compile: %w", err) + } + + logger.Info("Ignoring files with pattern `%s`", ignore) + + filteredStorage := storage.WithIgnoreFn(func(item absto.Item) bool { + if strings.HasPrefix(item.Pathname, provider.MetadataDirectoryName) { + return true + } + + if pattern.MatchString(item.Name) { + return true + } + + return false + }) + + return filteredStorage, nil +}