From f2aa5aad8af11d9dda9e1c2e1cf1a945778d6911 Mon Sep 17 00:00:00 2001 From: Vincent Boutour Date: Sat, 18 Mar 2023 14:30:42 +0100 Subject: [PATCH] chore(deps): Bumping absto to handle missing directory for s3 Signed-off-by: Vincent Boutour --- go.mod | 4 +-- go.sum | 8 ++--- pkg/crud/delete.go | 3 +- pkg/crud/get.go | 5 ++- pkg/crud/post.go | 3 +- pkg/crud/push.go | 21 ------------ pkg/crud/regenerate.go | 4 +-- pkg/crud/rename.go | 3 +- pkg/crud/upload.go | 2 +- pkg/crud/upload_chunk.go | 2 +- pkg/provider/event.go | 61 ++++++++++++++++++--------------- pkg/sanitizer/notify.go | 21 ------------ pkg/sanitizer/sanitizer.go | 70 +++++--------------------------------- 13 files changed, 56 insertions(+), 151 deletions(-) delete mode 100644 pkg/crud/push.go delete mode 100644 pkg/sanitizer/notify.go diff --git a/go.mod b/go.mod index 84e161d3..2c0e44eb 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/ViBiOh/ChatPotte v0.2.29 - github.com/ViBiOh/absto v1.3.2 + github.com/ViBiOh/absto v1.4.0 github.com/ViBiOh/auth/v2 v2.14.23 github.com/ViBiOh/exas v0.6.0 github.com/ViBiOh/flags v1.2.0 @@ -33,7 +33,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.0 // indirect + github.com/klauspost/compress v1.16.3 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/md5-simd v1.1.2 // indirect diff --git a/go.sum b/go.sum index 5fe46bc6..6ee0c3b7 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ViBiOh/ChatPotte v0.2.29 h1:8cIZDvkTfKOC4bZaQkxyOM47BcXZQR8ykJPl9LsZpMM= github.com/ViBiOh/ChatPotte v0.2.29/go.mod h1:GXEpgMQkN0R3CHpqVbuMhS8F6U1S/Qb7qQJ2juFqTtc= -github.com/ViBiOh/absto v1.3.2 h1:JZ6RJYh9YABO1QHfPcSjb/fEMjqcPcT59kH/jOr22Ys= -github.com/ViBiOh/absto v1.3.2/go.mod h1:RXbNesRB3Q9BWMxLeT3DXX29bhk5Tqifo2gssPtQVnk= +github.com/ViBiOh/absto v1.4.0 h1:fXaP7CbgQ/cJAfKM8Ksc6IlXkjWn6Dz6n9U6KHaQ5WA= +github.com/ViBiOh/absto v1.4.0/go.mod h1:PInUcncKy3FVbFkh2j2IvYJ906JS7HRMUEq+dArLSnw= github.com/ViBiOh/auth/v2 v2.14.23 h1:ZeQ2/tsh4Z3Jcu/y9R4X05uyhBUJeUC6UN8XyaKJ2s4= github.com/ViBiOh/auth/v2 v2.14.23/go.mod h1:pIMuBueO3sknvnvfAFJBOxTjoGWuxke6lp1ZSh6478w= github.com/ViBiOh/exas v0.6.0 h1:UWQuzesHShBSMkQw7mWAj4NBwL1Hb/fDcgzk/UpacYQ= @@ -175,8 +175,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= +github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= diff --git a/pkg/crud/delete.go b/pkg/crud/delete.go index c4650e5b..de3c9945 100644 --- a/pkg/crud/delete.go +++ b/pkg/crud/delete.go @@ -6,7 +6,6 @@ import ( "net/http" "github.com/ViBiOh/fibr/pkg/provider" - "github.com/ViBiOh/httputils/v4/pkg/cntxt" "github.com/ViBiOh/httputils/v4/pkg/model" "github.com/ViBiOh/httputils/v4/pkg/renderer" ) @@ -47,7 +46,7 @@ func (a App) Delete(w http.ResponseWriter, r *http.Request, request provider.Req provider.SetPrefsCookie(w, request) } - go a.notify(cntxt.WithoutDeadline(ctx), provider.NewDeleteEvent(request, info, a.rendererApp)) + go a.pushEvent(provider.NewDeleteEvent(ctx, request, info, a.rendererApp)) a.rendererApp.Redirect(w, r, fmt.Sprintf("?d=%s", request.Display), renderer.NewSuccessMessage("%s successfully deleted", info.Name)) } diff --git a/pkg/crud/get.go b/pkg/crud/get.go index a9d73238..ef0d7579 100644 --- a/pkg/crud/get.go +++ b/pkg/crud/get.go @@ -16,7 +16,6 @@ import ( "github.com/ViBiOh/fibr/pkg/geo" "github.com/ViBiOh/fibr/pkg/metadata" "github.com/ViBiOh/fibr/pkg/provider" - "github.com/ViBiOh/httputils/v4/pkg/cntxt" "github.com/ViBiOh/httputils/v4/pkg/logger" "github.com/ViBiOh/httputils/v4/pkg/model" "github.com/ViBiOh/httputils/v4/pkg/query" @@ -75,7 +74,7 @@ func (a App) handleFile(w http.ResponseWriter, r *http.Request, request provider if query.GetBool(r, "browser") { provider.SetPrefsCookie(w, request) - go a.notify(cntxt.WithoutDeadline(r.Context()), provider.NewAccessEvent(item, r)) + go a.pushEvent(provider.NewAccessEvent(r.Context(), item, r)) return a.browse(r.Context(), request, item, message) } @@ -132,7 +131,7 @@ func (a App) handleDir(w http.ResponseWriter, r *http.Request, request provider. return errorReturn(request, err) } - go a.notify(cntxt.WithoutDeadline(r.Context()), provider.NewAccessEvent(item, r)) + go a.pushEvent(provider.NewAccessEvent(r.Context(), item, r)) if query.GetBool(r, "search") { return a.search(r, request, item, items) diff --git a/pkg/crud/post.go b/pkg/crud/post.go index 218de1a3..9a7fde84 100644 --- a/pkg/crud/post.go +++ b/pkg/crud/post.go @@ -10,7 +10,6 @@ import ( "strings" "github.com/ViBiOh/fibr/pkg/provider" - "github.com/ViBiOh/httputils/v4/pkg/cntxt" "github.com/ViBiOh/httputils/v4/pkg/model" "github.com/ViBiOh/httputils/v4/pkg/renderer" ) @@ -175,7 +174,7 @@ func (a App) handlePostDescription(w http.ResponseWriter, r *http.Request, reque return } - go a.notify(cntxt.WithoutDeadline(ctx), provider.NewDescriptionEvent(item, a.bestSharePath(item.Pathname), description, a.rendererApp)) + go a.pushEvent(provider.NewDescriptionEvent(ctx, item, a.bestSharePath(item.Pathname), description, a.rendererApp)) a.rendererApp.Redirect(w, r, fmt.Sprintf("?d=%s#%s", request.Display, item.ID), renderer.NewSuccessMessage("Description successfully edited")) } diff --git a/pkg/crud/push.go b/pkg/crud/push.go deleted file mode 100644 index 04612faf..00000000 --- a/pkg/crud/push.go +++ /dev/null @@ -1,21 +0,0 @@ -package crud - -import ( - "context" - - "github.com/ViBiOh/fibr/pkg/provider" - "github.com/ViBiOh/httputils/v4/pkg/logger" - "go.opentelemetry.io/otel/trace" -) - -func (a App) notify(ctx context.Context, event provider.Event) { - if a.pushEvent == nil { - return - } - - event.TraceLink = trace.LinkFromContext(ctx) - - if err := a.pushEvent(event); err != nil { - logger.Error("push event %+v: %s", event, err) - } -} diff --git a/pkg/crud/regenerate.go b/pkg/crud/regenerate.go index dc49513d..7c002bb3 100644 --- a/pkg/crud/regenerate.go +++ b/pkg/crud/regenerate.go @@ -42,7 +42,7 @@ func (a App) regenerate(w http.ResponseWriter, r *http.Request, request provider if item.IsDir { directories = append(directories, item) } else { - a.notify(ctx, provider.NewRestartEvent(item, subset)) + a.pushEvent(provider.NewRestartEvent(ctx, item, subset)) } return nil @@ -52,7 +52,7 @@ func (a App) regenerate(w http.ResponseWriter, r *http.Request, request provider } for _, directory := range directories { - a.notify(ctx, provider.NewStartEvent(directory)) + a.pushEvent(provider.NewStartEvent(ctx, directory)) } }(cntxt.WithoutDeadline(ctx)) diff --git a/pkg/crud/rename.go b/pkg/crud/rename.go index 12478910..a6afb045 100644 --- a/pkg/crud/rename.go +++ b/pkg/crud/rename.go @@ -9,7 +9,6 @@ import ( absto "github.com/ViBiOh/absto/pkg/model" "github.com/ViBiOh/fibr/pkg/provider" - "github.com/ViBiOh/httputils/v4/pkg/cntxt" "github.com/ViBiOh/httputils/v4/pkg/model" "github.com/ViBiOh/httputils/v4/pkg/renderer" ) @@ -24,7 +23,7 @@ func (a App) DoRename(ctx context.Context, oldPath, newPath string, oldItem abst return absto.Item{}, fmt.Errorf("get info of new item: %w", err) } - go a.notify(cntxt.WithoutDeadline(ctx), provider.NewRenameEvent(oldItem, newItem, a.bestSharePath(newPath), a.rendererApp)) + go a.pushEvent(provider.NewRenameEvent(ctx, oldItem, newItem, a.bestSharePath(newPath), a.rendererApp)) return newItem, nil } diff --git a/pkg/crud/upload.go b/pkg/crud/upload.go index 3fff0aa3..01cb7fef 100644 --- a/pkg/crud/upload.go +++ b/pkg/crud/upload.go @@ -38,7 +38,7 @@ func (a App) saveUploadedFile(ctx context.Context, request provider.Request, inp if info, infoErr := a.storageApp.Info(ctx, filePath); infoErr != nil { logger.Error("get info for upload event: %s", infoErr) } else { - a.notify(ctx, provider.NewUploadEvent(request, info, a.bestSharePath(filePath), a.rendererApp)) + a.pushEvent(provider.NewUploadEvent(ctx, request, info, a.bestSharePath(filePath), a.rendererApp)) } }(cntxt.WithoutDeadline(ctx)) } diff --git a/pkg/crud/upload_chunk.go b/pkg/crud/upload_chunk.go index 441e7cfb..9b49f727 100644 --- a/pkg/crud/upload_chunk.go +++ b/pkg/crud/upload_chunk.go @@ -113,7 +113,7 @@ func (a App) mergeChunk(w http.ResponseWriter, r *http.Request, request provider if info, infoErr := a.storageApp.Info(ctx, filePath); infoErr != nil { logger.Error("get info for upload event: %s", infoErr) } else { - a.notify(ctx, provider.NewUploadEvent(request, info, a.bestSharePath(filePath), a.rendererApp)) + a.pushEvent(provider.NewUploadEvent(ctx, request, info, a.bestSharePath(filePath), a.rendererApp)) } }(cntxt.WithoutDeadline(ctx)) } diff --git a/pkg/provider/event.go b/pkg/provider/event.go index 747c8dcf..bfc9a584 100644 --- a/pkg/provider/event.go +++ b/pkg/provider/event.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "net/http" "path" @@ -22,7 +21,7 @@ import ( type EventType uint -type EventProducer func(Event) error +type EventProducer func(Event) type EventConsumer func(context.Context, Event) @@ -167,11 +166,12 @@ func (e Event) GetTo() string { return newName } -func NewUploadEvent(request Request, item absto.Item, shareableURL string, rendererApp renderer.App) Event { +func NewUploadEvent(ctx context.Context, request Request, item absto.Item, shareableURL string, rendererApp renderer.App) Event { return Event{ Time: time.Now(), Type: UploadEvent, Item: item, + TraceLink: trace.LinkFromContext(ctx), URL: rendererApp.PublicURL(request.AbsoluteURL(item.Name)), ShareableURL: rendererApp.PublicURL(shareableURL), Metadata: map[string]string{ @@ -180,22 +180,24 @@ func NewUploadEvent(request Request, item absto.Item, shareableURL string, rende } } -func NewRenameEvent(old, new absto.Item, shareableURL string, rendererApp renderer.App) Event { +func NewRenameEvent(ctx context.Context, old, new absto.Item, shareableURL string, rendererApp renderer.App) Event { return Event{ Time: time.Now(), Type: RenameEvent, Item: old, + TraceLink: trace.LinkFromContext(ctx), New: &new, URL: rendererApp.PublicURL(new.Pathname), ShareableURL: rendererApp.PublicURL(shareableURL), } } -func NewDescriptionEvent(item absto.Item, shareableURL string, description string, rendererApp renderer.App) Event { +func NewDescriptionEvent(ctx context.Context, item absto.Item, shareableURL string, description string, rendererApp renderer.App) Event { return Event{ Time: time.Now(), Type: DescriptionEvent, Item: item, + TraceLink: trace.LinkFromContext(ctx), URL: rendererApp.PublicURL(item.Pathname), ShareableURL: rendererApp.PublicURL(shareableURL), Metadata: map[string]string{ @@ -204,35 +206,38 @@ func NewDescriptionEvent(item absto.Item, shareableURL string, description strin } } -func NewDeleteEvent(request Request, item absto.Item, rendererApp renderer.App) Event { +func NewDeleteEvent(ctx context.Context, request Request, item absto.Item, rendererApp renderer.App) Event { return Event{ - Time: time.Now(), - Type: DeleteEvent, - Item: item, - URL: rendererApp.PublicURL(request.AbsoluteURL("")), + Time: time.Now(), + Type: DeleteEvent, + Item: item, + TraceLink: trace.LinkFromContext(ctx), + URL: rendererApp.PublicURL(request.AbsoluteURL("")), } } -func NewStartEvent(item absto.Item) Event { +func NewStartEvent(ctx context.Context, item absto.Item) Event { return Event{ - Time: time.Now(), - Type: StartEvent, - Item: item, + Time: time.Now(), + Type: StartEvent, + Item: item, + TraceLink: trace.LinkFromContext(ctx), } } -func NewRestartEvent(item absto.Item, subset string) Event { +func NewRestartEvent(ctx context.Context, item absto.Item, subset string) Event { return Event{ - Time: time.Now(), - Type: StartEvent, - Item: item, + Time: time.Now(), + Type: StartEvent, + Item: item, + TraceLink: trace.LinkFromContext(ctx), Metadata: map[string]string{ "force": subset, }, } } -func NewAccessEvent(item absto.Item, r *http.Request) Event { +func NewAccessEvent(ctx context.Context, item absto.Item, r *http.Request) Event { metadata := make(map[string]string) for key, values := range r.Header { if strings.EqualFold(key, "Authorization") { @@ -246,11 +251,12 @@ func NewAccessEvent(item absto.Item, r *http.Request) Event { metadata["URL"] = r.URL.String() return Event{ - Time: time.Now(), - Type: AccessEvent, - Item: item, - Metadata: metadata, - URL: r.URL.String(), + Time: time.Now(), + Type: AccessEvent, + Item: item, + TraceLink: trace.LinkFromContext(ctx), + Metadata: metadata, + URL: r.URL.String(), } } @@ -298,21 +304,20 @@ func (e EventBus) Done() <-chan struct{} { return e.done } -func (e EventBus) Push(event Event) error { +func (e EventBus) Push(event Event) { select { case <-e.closed: e.increaseMetric(event, "refused") - return errors.New("bus is closed") + logger.Error("bus is closed") default: } select { case <-e.closed: e.increaseMetric(event, "refused") - return errors.New("bus is closed") + logger.Error("bus is closed") case e.bus <- event: e.increaseMetric(event, "push") - return nil } } diff --git a/pkg/sanitizer/notify.go b/pkg/sanitizer/notify.go deleted file mode 100644 index f01c2b38..00000000 --- a/pkg/sanitizer/notify.go +++ /dev/null @@ -1,21 +0,0 @@ -package sanitizer - -import ( - "context" - - "github.com/ViBiOh/fibr/pkg/provider" - "github.com/ViBiOh/httputils/v4/pkg/logger" - "go.opentelemetry.io/otel/trace" -) - -func (a App) notify(ctx context.Context, event provider.Event) { - if a.pushEvent == nil { - return - } - - event.TraceLink = trace.LinkFromContext(ctx) - - if err := a.pushEvent(event); err != nil { - logger.Error("push event %+v: %s", event, err) - } -} diff --git a/pkg/sanitizer/sanitizer.go b/pkg/sanitizer/sanitizer.go index f05619b0..646ce38e 100644 --- a/pkg/sanitizer/sanitizer.go +++ b/pkg/sanitizer/sanitizer.go @@ -30,18 +30,6 @@ type Config struct { sanitizeOnStart *bool } -type Items []absto.Item - -func (i Items) FindDirectory(name string) (absto.Item, bool) { - for _, item := range i { - if item.IsDir && absto.Dirname(item.Pathname) == name { - return item, true - } - } - - return absto.Item{}, false -} - func Flags(fs *flag.FlagSet, prefix string, overrides ...flags.Override) Config { return Config{ sanitizeOnStart: flags.Bool(fs, prefix, "crud", "SanitizeOnStart", "Sanitize on start", false, nil), @@ -67,20 +55,19 @@ func (a App) Start(ctx context.Context) { defer close(a.done) if err := a.exclusiveApp.Execute(ctx, "fibr:mutex:start", time.Hour, func(ctx context.Context) error { - a.start(ctx) - return nil + return a.start(ctx) }); err != nil { logger.Error("start: %s", err) } } -func (a App) start(ctx context.Context) { +func (a App) start(ctx context.Context) error { logger.Info("Starting startup check...") defer logger.Info("Ending startup check.") done := ctx.Done() - var directories Items + var directories []absto.Item err := a.storageApp.Walk(ctx, "", func(item absto.Item) error { select { @@ -94,22 +81,20 @@ func (a App) start(ctx context.Context) { if item.IsDir { directories = append(directories, item) } else { - a.notify(ctx, provider.NewStartEvent(item)) - - if created := a.sanitizeOrphan(ctx, directories, item); !created.IsZero() { - directories = append(directories, created) - } + a.pushEvent(provider.NewStartEvent(ctx, item)) } return nil }) if err != nil { - logger.Error("start: %s", err) + return err } for _, directory := range directories { - a.notify(ctx, provider.NewStartEvent(directory)) + a.pushEvent(provider.NewStartEvent(ctx, directory)) } + + return nil } func (a App) sanitizeName(ctx context.Context, item absto.Item) absto.Item { @@ -128,45 +113,6 @@ func (a App) sanitizeName(ctx context.Context, item absto.Item) absto.Item { return item } - return a.rename(ctx, item, name) -} - -func (a App) sanitizeOrphan(ctx context.Context, directories Items, item absto.Item) absto.Item { - dirname := item.Dir() - - _, ok := directories.FindDirectory(dirname) - if ok { - return absto.Item{} - } - - if !a.sanitizeOnStart { - logger.Warn("File with name `%s` doesn't have a parent directory", item.Pathname) - return absto.Item{} - } - - sanitizedName, err := provider.SanitizeName(dirname, false) - if err != nil { - logger.Error("sanitize name for directory `%s`: %s", dirname, err) - return absto.Item{} - } - - logger.Info("Creating folder `%s`", sanitizedName) - - if err := a.storageApp.CreateDir(ctx, sanitizedName); err != nil { - logger.Error("create a parent directory for `%s`: %s", item.Pathname, err) - return absto.Item{} - } - - directoryItem, err := a.storageApp.Info(ctx, sanitizedName) - if err != nil { - logger.Error("getting the parent directory infos `%s`: %s", item.Pathname, err) - return absto.Item{} - } - - return directoryItem -} - -func (a App) rename(ctx context.Context, item absto.Item, name string) absto.Item { logger.Info("Renaming `%s` to `%s`", item.Pathname, name) renamedItem, err := a.renamer.DoRename(ctx, item.Pathname, name, item)