From 19645db2de839cd4de64e956e79bf19793622f8a Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 22 Jun 2024 23:48:07 +1200 Subject: [PATCH 1/8] Use correct `AS` case --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fa7f27fe1..c10460944 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:alpine as builder +FROM golang:alpine AS builder ARG VERSION=dev From c7e04554791102b3e041ae9da3f9e11bb00b320f Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 22 Jun 2024 23:56:17 +1200 Subject: [PATCH 2/8] Handle errors correctly --- internal/storage/messages.go | 4 +++- internal/storage/schemas.go | 4 +++- server/pop3/pop3_test.go | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/storage/messages.go b/internal/storage/messages.go index 638267232..9af7b5f87 100644 --- a/internal/storage/messages.go +++ b/internal/storage/messages.go @@ -593,7 +593,9 @@ func DeleteMessages(ids []string) error { } } - err = tx.Commit() + if err := tx.Commit(); err != nil { + return err + } dbLastAction = time.Now() addDeletedSize(int64(totalSize)) diff --git a/internal/storage/schemas.go b/internal/storage/schemas.go index 874e45c54..7b686786d 100644 --- a/internal/storage/schemas.go +++ b/internal/storage/schemas.go @@ -137,7 +137,9 @@ func dbApplySchemas() error { buf := new(bytes.Buffer) - err = t1.Execute(buf, nil) + if err := t1.Execute(buf, nil); err != nil { + return err + } if _, err := db.Exec(buf.String()); err != nil { return err diff --git a/server/pop3/pop3_test.go b/server/pop3/pop3_test.go index 912ab3a30..dd1672c1e 100644 --- a/server/pop3/pop3_test.go +++ b/server/pop3/pop3_test.go @@ -67,7 +67,7 @@ func TestPOP3(t *testing.T) { return } - count, size, err = c.Stat() + count, _, err = c.Stat() if err != nil { t.Errorf(err.Error()) return From 0dca8df29cb0c2f0664ea3c99ed17566083c9fea Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Fri, 28 Jun 2024 22:35:07 +1200 Subject: [PATCH 3/8] Feature: Add option to disable auto-tagging for plus-addresses & X-Tags (#323) --- cmd/root.go | 2 ++ config/config.go | 9 ++++++++- config/tags.go | 30 ++++++++++++++++++++++++++++++ internal/storage/messages.go | 32 ++++++++++++++++++-------------- 4 files changed, 58 insertions(+), 15 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index ccdc9502a..a17d16c3e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -131,6 +131,7 @@ func init() { rootCmd.Flags().StringVarP(&config.CLITagsArg, "tag", "t", config.CLITagsArg, "Tag new messages matching filters") rootCmd.Flags().StringVar(&config.TagsConfig, "tags-config", config.TagsConfig, "Load tags filters from yaml configuration file") rootCmd.Flags().BoolVar(&tools.TagsTitleCase, "tags-title-case", tools.TagsTitleCase, "TitleCase new tags generated from plus-addresses and X-Tags") + rootCmd.Flags().StringVar(&config.TagsDisable, "tags-disable", config.TagsDisable, "Disable auto-tagging, comma separated (eg: plus-addresses,x-tags)") // Webhook rootCmd.Flags().StringVar(&config.WebhookURL, "webhook-url", config.WebhookURL, "Send a webhook request for new messages") @@ -290,6 +291,7 @@ func initConfigFromEnv() { config.CLITagsArg = os.Getenv("MP_TAG") config.TagsConfig = os.Getenv("MP_TAGS_CONFIG") tools.TagsTitleCase = getEnabledFromEnv("MP_TAGS_TITLE_CASE") + config.TagsDisable = os.Getenv("MP_TAGS_DISABLE") // Webhook if len(os.Getenv("MP_WEBHOOK_URL")) > 0 { diff --git a/config/config.go b/config/config.go index ceb80bf73..c5c914b97 100644 --- a/config/config.go +++ b/config/config.go @@ -102,6 +102,10 @@ var ( // TagFilters are used to apply tags to new mail TagFilters []autoTag + // TagsDisable accepts a comma-separated list of tag types to disable + // including x-tags & plus-addresses + TagsDisable string + // SMTPRelayConfigFile to parse a yaml file and store config of relay SMTP server SMTPRelayConfigFile string @@ -390,7 +394,7 @@ func VerifyConfig() error { } } - // load tag filters + // load tag filters & options TagFilters = []autoTag{} if err := loadTagsFromArgs(CLITagsArg); err != nil { return err @@ -398,6 +402,9 @@ func VerifyConfig() error { if err := loadTagsFromConfig(TagsConfig); err != nil { return err } + if err := parseTagsDisable(TagsDisable); err != nil { + return err + } if SMTPAllowedRecipients != "" { restrictRegexp, err := regexp.Compile(SMTPAllowedRecipients) diff --git a/config/tags.go b/config/tags.go index cb46f2599..c11ec26a3 100644 --- a/config/tags.go +++ b/config/tags.go @@ -11,6 +11,14 @@ import ( "gopkg.in/yaml.v3" ) +var ( + // TagsDisablePlus disables message tagging using plus-addresses (user+tag@example.com) - set via verifyConfig() + TagsDisablePlus bool + + // TagsDisableXTags disables message tagging via the X-Tags header - set via verifyConfig() + TagsDisableXTags bool +) + type yamlTags struct { Filters []yamlTag `yaml:"filters"` } @@ -79,3 +87,25 @@ func loadTagsFromArgs(c string) error { return nil } + +func parseTagsDisable(s string) error { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + + parts := strings.Split(strings.ToLower(s), ",") + + for _, p := range parts { + switch strings.TrimSpace(p) { + case "x-tags", "xtags": + TagsDisableXTags = true + case "plus-addresses", "plus-addressing": + TagsDisablePlus = true + default: + return fmt.Errorf("[tags] invalid --tags-disable option: %s", p) + } + } + + return nil +} diff --git a/internal/storage/messages.go b/internal/storage/messages.go index 9af7b5f87..a703f0837 100644 --- a/internal/storage/messages.go +++ b/internal/storage/messages.go @@ -112,20 +112,24 @@ func Store(body *[]byte) (string, error) { return "", err } - // extract tags from body matches - rawTags := findTagsInRawMessage(body) - // extract plus addresses tags from enmime.Envelope - plusTags := obj.tagsFromPlusAddresses() - // extract tags from X-Tags header - xTags := tools.SetTagCasing(strings.Split(strings.TrimSpace(env.Root.Header.Get("X-Tags")), ",")) - // extract tags from search matches - searchTags := tagFilterMatches(id) - - // combine all tags into one slice - tags := append(rawTags, plusTags...) - tags = append(tags, xTags...) - // sort and extract only unique tags - tags = sortedUniqueTags(append(tags, searchTags...)) + // extract tags using pre-set tag filters, empty slice if not set + tags := findTagsInRawMessage(body) + + if !config.TagsDisableXTags { + xTagsHdr := env.Root.Header.Get("X-Tags") + if xTagsHdr != "" { + // extract tags from X-Tags header + tags = append(tags, tools.SetTagCasing(strings.Split(strings.TrimSpace(xTagsHdr), ","))...) + } + } + + if !config.TagsDisablePlus { + // get tags from plus-addresses + tags = append(tags, obj.tagsFromPlusAddresses()...) + } + + // extract tags from search matches, and sort and extract unique tags + tags = sortedUniqueTags(append(tags, tagFilterMatches(id)...)) if len(tags) > 0 { if err := SetMessageTags(id, tags); err != nil { From 0c377b9616673cdb5136a9aefdc53bcbfca7cda3 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 29 Jun 2024 17:12:56 +1200 Subject: [PATCH 4/8] Feature: Add ability to rename and delete tags globally --- internal/storage/messages.go | 6 +- internal/storage/schemas.go | 2 +- internal/storage/tags.go | 106 ++++++++++++++++++++--- internal/storage/tags_test.go | 8 +- server/apiv1/api.go | 2 +- server/apiv1/swagger.go | 16 ++++ server/apiv1/tags.go | 100 ++++++++++++++++++++++ server/pop3/pop3_test.go | 2 +- server/server.go | 2 + server/server_test.go | 2 +- server/ui-src/App.vue | 3 + server/ui-src/components/EditTags.vue | 119 ++++++++++++++++++++++++++ server/ui-src/components/NavTags.vue | 5 ++ server/ui/api/v1/swagger.json | 88 +++++++++++++++++++ 14 files changed, 438 insertions(+), 23 deletions(-) create mode 100644 server/apiv1/tags.go create mode 100644 server/ui-src/components/EditTags.vue diff --git a/internal/storage/messages.go b/internal/storage/messages.go index a703f0837..d4e139ec6 100644 --- a/internal/storage/messages.go +++ b/internal/storage/messages.go @@ -131,8 +131,10 @@ func Store(body *[]byte) (string, error) { // extract tags from search matches, and sort and extract unique tags tags = sortedUniqueTags(append(tags, tagFilterMatches(id)...)) + setTags := []string{} if len(tags) > 0 { - if err := SetMessageTags(id, tags); err != nil { + setTags, err = SetMessageTags(id, tags) + if err != nil { return "", err } } @@ -148,7 +150,7 @@ func Store(body *[]byte) (string, error) { c.Attachments = attachments c.Subject = subject c.Size = size - c.Tags = tags + c.Tags = setTags c.Snippet = snippet websockets.Broadcast("new", c) diff --git a/internal/storage/schemas.go b/internal/storage/schemas.go index 7b686786d..aaf26080f 100644 --- a/internal/storage/schemas.go +++ b/internal/storage/schemas.go @@ -199,7 +199,7 @@ func migrateTagsToManyMany() { if len(toConvert) > 0 { logger.Log().Infof("[migration] converting %d message tags", len(toConvert)) for id, tags := range toConvert { - if err := SetMessageTags(id, tags); err != nil { + if _, err := SetMessageTags(id, tags); err != nil { logger.Log().Errorf("[migration] %s", err.Error()) } else { if _, err := sqlf.Update(tenant("mailbox")). diff --git a/internal/storage/tags.go b/internal/storage/tags.go index 9facfc4fe..c87189270 100644 --- a/internal/storage/tags.go +++ b/internal/storage/tags.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "fmt" "regexp" "sort" "strings" @@ -21,7 +22,7 @@ var ( ) // SetMessageTags will set the tags for a given database ID, removing any not in the array -func SetMessageTags(id string, tags []string) error { +func SetMessageTags(id string, tags []string) ([]string, error) { applyTags := []string{} for _, t := range tags { t = tools.CleanTag(t) @@ -30,6 +31,7 @@ func SetMessageTags(id string, tags []string) error { } } + tagNames := []string{} currentTags := getMessageTags(id) origTagCount := len(currentTags) @@ -38,9 +40,12 @@ func SetMessageTags(id string, tags []string) error { continue } - if err := AddMessageTag(id, t); err != nil { - return err + name, err := AddMessageTag(id, t) + if err != nil { + return []string{}, err } + + tagNames = append(tagNames, name) } if origTagCount > 0 { @@ -49,42 +54,44 @@ func SetMessageTags(id string, tags []string) error { for _, t := range currentTags { if !tools.InArray(t, applyTags) { if err := DeleteMessageTag(id, t); err != nil { - return err + return []string{}, err } } } } - return nil + return tagNames, nil } // AddMessageTag adds a tag to a message -func AddMessageTag(id, name string) error { +func AddMessageTag(id, name string) (string, error) { // prevent two identical tags being added at the same time addTagMutex.Lock() var tagID int + var foundName sql.NullString q := sqlf.From(tenant("tags")). Select("ID").To(&tagID). + Select("Name").To(&foundName). Where("Name = ?", name) // if tag exists - add tag to message if err := q.QueryRowAndClose(context.TODO(), db); err == nil { addTagMutex.Unlock() // check message does not already have this tag - var count int + var exists int if err := sqlf.From(tenant("message_tags")). - Select("COUNT(ID)").To(&count). + Select("COUNT(ID)").To(&exists). Where("ID = ?", id). Where("TagID = ?", tagID). QueryRowAndClose(context.Background(), db); err != nil { - return err + return "", err } - if count > 0 { + if exists > 0 { // already exists - return nil + return foundName.String, nil } logger.Log().Debugf("[tags] adding tag \"%s\" to %s", name, id) @@ -93,7 +100,7 @@ func AddMessageTag(id, name string) error { Set("ID", id). Set("TagID", tagID). ExecAndClose(context.TODO(), db) - return err + return foundName.String, err } // new tag, add to the database @@ -101,7 +108,7 @@ func AddMessageTag(id, name string) error { Set("Name", name). ExecAndClose(context.TODO(), db); err != nil { addTagMutex.Unlock() - return err + return name, err } addTagMutex.Unlock() @@ -174,6 +181,79 @@ func GetAllTagsCount() map[string]int64 { return tags } +// RenameTag renames a tag +func RenameTag(from, to string) error { + to = tools.CleanTag(to) + if to == "" || !config.ValidTagRegexp.MatchString(to) { + return fmt.Errorf("invalid tag name: %s", to) + } + + if from == to { + return nil // ignore + } + + var id, existsID int + + q := sqlf.From(tenant("tags")). + Select(`ID`).To(&id). + Where(`Name = ?`, from). + Limit(1) + err := q.QueryRowAndClose(context.Background(), db) + if err != nil { + return fmt.Errorf("tag not found: %s", from) + } + + // check if another tag by this name already exists + q = sqlf.From(tenant("tags")). + Select("ID").To(&existsID). + Where(`Name = ?`, to). + Where(`ID != ?`, id). + Limit(1) + err = q.QueryRowAndClose(context.Background(), db) + if err == nil || existsID != 0 { + return fmt.Errorf("tag already exists: %s", to) + } + + q = sqlf.Update(tenant("tags")). + Set("Name", to). + Where("ID = ?", id) + _, err = q.ExecAndClose(context.Background(), db) + + return err +} + +// DeleteTag deleted a tag and removed all references to the tag +func DeleteTag(tag string) error { + var id int + + q := sqlf.From(tenant("tags")). + Select(`ID`).To(&id). + Where(`Name = ?`, tag). + Limit(1) + err := q.QueryRowAndClose(context.Background(), db) + if err != nil { + return fmt.Errorf("tag not found: %s", tag) + } + + // delete all references + q = sqlf.DeleteFrom(tenant("message_tags")). + Where(`TagID = ?`, id) + _, err = q.ExecAndClose(context.Background(), db) + if err != nil { + return fmt.Errorf("error deleting tag references: %s", err.Error()) + } + + // delete tag + q = sqlf.DeleteFrom(tenant("tags")). + Where(`ID = ?`, id) + _, err = q.ExecAndClose(context.Background(), db) + if err != nil { + return fmt.Errorf("error deleting tag: %s", err.Error()) + } + + return nil +} + // PruneUnusedTags will delete all unused tags from the database func pruneUnusedTags() error { q := sqlf.From(tenant("tags")). diff --git a/internal/storage/tags_test.go b/internal/storage/tags_test.go index 388fe8b93..7b3636a58 100644 --- a/internal/storage/tags_test.go +++ b/internal/storage/tags_test.go @@ -24,7 +24,7 @@ func TestTags(t *testing.T) { } for i := 0; i < 10; i++ { - if err := SetMessageTags(ids[i], []string{fmt.Sprintf("Tag-%d", i)}); err != nil { + if _, err := SetMessageTags(ids[i], []string{fmt.Sprintf("Tag-%d", i)}); err != nil { t.Log("error ", err) t.Fail() } @@ -58,7 +58,7 @@ func TestTags(t *testing.T) { // pad number with 0 to ensure they are returned alphabetically newTags = append(newTags, fmt.Sprintf("AnotherTag %02d", i)) } - if err := SetMessageTags(id, newTags); err != nil { + if _, err := SetMessageTags(id, newTags); err != nil { t.Log("error ", err) t.Fail() } @@ -82,7 +82,7 @@ func TestTags(t *testing.T) { assertEqual(t, "", strings.Join(returnedTags, "|"), "Message tags should be empty") // apply the same tag twice - if err := SetMessageTags(id, []string{"Duplicate Tag", "Duplicate Tag"}); err != nil { + if _, err := SetMessageTags(id, []string{"Duplicate Tag", "Duplicate Tag"}); err != nil { t.Log("error ", err) t.Fail() } @@ -94,7 +94,7 @@ func TestTags(t *testing.T) { } // apply tag with invalid characters - if err := SetMessageTags(id, []string{"Dirty! \"Tag\""}); err != nil { + if _, err := SetMessageTags(id, []string{"Dirty! \"Tag\""}); err != nil { t.Log("error ", err) t.Fail() } diff --git a/server/apiv1/api.go b/server/apiv1/api.go index 905bd5791..f57a78e14 100644 --- a/server/apiv1/api.go +++ b/server/apiv1/api.go @@ -583,7 +583,7 @@ func SetMessageTags(w http.ResponseWriter, r *http.Request) { if len(ids) > 0 { for _, id := range ids { - if err := storage.SetMessageTags(id, data.Tags); err != nil { + if _, err := storage.SetMessageTags(id, data.Tags); err != nil { httpError(w, err.Error()) return } diff --git a/server/apiv1/swagger.go b/server/apiv1/swagger.go index b0c5c3005..d9977965c 100644 --- a/server/apiv1/swagger.go +++ b/server/apiv1/swagger.go @@ -95,6 +95,22 @@ type setTagsRequestBody struct { IDs []string } +// swagger:parameters RenameTag +type renameTagParams struct { + // in: body + Body *renameTagRequestBody +} + +// Rename tag request +// swagger:model renameTagRequestBody +type renameTagRequestBody struct { + // New name + // + // required: true + // example: New name + Name string +} + // swagger:parameters ReleaseMessage type releaseMessageParams struct { // Message database ID diff --git a/server/apiv1/tags.go b/server/apiv1/tags.go new file mode 100644 index 000000000..7eef44877 --- /dev/null +++ b/server/apiv1/tags.go @@ -0,0 +1,100 @@ +package apiv1 + +import ( + "encoding/json" + "net/http" + + "github.com/axllent/mailpit/internal/storage" + "github.com/axllent/mailpit/server/websockets" + "github.com/gorilla/mux" +) + +// RenameTag (method: PUT) used to rename a tag +func RenameTag(w http.ResponseWriter, r *http.Request) { + // swagger:route PUT /api/v1/tags/{tag} tags RenameTag + // + // # Rename a tag + // + // Renames a tag. + // + // Produces: + // - text/plain + // + // Schemes: http, https + // + // Parameters: + // + name: tag + // in: path + // description: The url-encoded tag name to rename + // required: true + // type: string + // + // Responses: + // 200: OKResponse + // default: ErrorResponse + + vars := mux.Vars(r) + + tag := vars["tag"] + + decoder := json.NewDecoder(r.Body) + + var data struct { + Name string + } + + err := decoder.Decode(&data) + if err != nil { + httpError(w, err.Error()) + return + } + + if err := storage.RenameTag(tag, data.Name); err != nil { + httpError(w, err.Error()) + return + } + + websockets.Broadcast("prune", nil) + + w.Header().Add("Content-Type", "text/plain") + _, _ = w.Write([]byte("ok")) +} + +// DeleteTag (method: DELETE) used to delete a tag +func DeleteTag(w http.ResponseWriter, r *http.Request) { + // swagger:route DELETE /api/v1/tags/{tag} tags DeleteTag + // + // # Delete a tag + // + // Deletes a tag. This will not delete any messages with this tag. + // + // Produces: + // - text/plain + // + // Schemes: http, https + // + // Parameters: + // + name: tag + // in: path + // description: The url-encoded tag name to delete + // required: true + // type: string + // + // Responses: + // 200: OKResponse + // default: ErrorResponse + + vars := mux.Vars(r) + + tag := vars["tag"] + + if err := storage.DeleteTag(tag); err != nil { + httpError(w, err.Error()) + return + } + + websockets.Broadcast("prune", nil) + + w.Header().Add("Content-Type", "text/plain") + _, _ = w.Write([]byte("ok")) +} diff --git a/server/pop3/pop3_test.go b/server/pop3/pop3_test.go index dd1672c1e..9012becdd 100644 --- a/server/pop3/pop3_test.go +++ b/server/pop3/pop3_test.go @@ -349,7 +349,7 @@ func insertEmailData(t *testing.T) { t.Fail() } - if err := storage.SetMessageTags(id, []string{fmt.Sprintf("Test tag %03d", i)}); err != nil { + if _, err := storage.SetMessageTags(id, []string{fmt.Sprintf("Test tag %03d", i)}); err != nil { t.Log("error ", err) t.Fail() } diff --git a/server/server.go b/server/server.go index 8ea160db9..aee94d59d 100644 --- a/server/server.go +++ b/server/server.go @@ -132,6 +132,8 @@ func apiRoutes() *mux.Router { r.HandleFunc(config.Webroot+"api/v1/send", middleWareFunc(apiv1.SendMessageHandler)).Methods("POST") r.HandleFunc(config.Webroot+"api/v1/tags", middleWareFunc(apiv1.GetAllTags)).Methods("GET") r.HandleFunc(config.Webroot+"api/v1/tags", middleWareFunc(apiv1.SetMessageTags)).Methods("PUT") + r.HandleFunc(config.Webroot+"api/v1/tags/{tag}", middleWareFunc(apiv1.RenameTag)).Methods("PUT") + r.HandleFunc(config.Webroot+"api/v1/tags/{tag}", middleWareFunc(apiv1.DeleteTag)).Methods("DELETE") r.HandleFunc(config.Webroot+"api/v1/message/{id}/part/{partID}", middleWareFunc(apiv1.DownloadAttachment)).Methods("GET") r.HandleFunc(config.Webroot+"api/v1/message/{id}/part/{partID}/thumb", middleWareFunc(apiv1.Thumbnail)).Methods("GET") r.HandleFunc(config.Webroot+"api/v1/message/{id}/headers", middleWareFunc(apiv1.GetHeaders)).Methods("GET") diff --git a/server/server_test.go b/server/server_test.go index b39b0a63c..da9a53e34 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -383,7 +383,7 @@ func insertEmailData(t *testing.T) { t.Fail() } - if err := storage.SetMessageTags(id, []string{fmt.Sprintf("Test tag %03d", i)}); err != nil { + if _, err := storage.SetMessageTags(id, []string{fmt.Sprintf("Test tag %03d", i)}); err != nil { t.Log("error ", err) t.Fail() } diff --git a/server/ui-src/App.vue b/server/ui-src/App.vue index 35e336a3e..913f4db6b 100644 --- a/server/ui-src/App.vue +++ b/server/ui-src/App.vue @@ -2,6 +2,7 @@ import CommonMixins from './mixins/CommonMixins' import Favicon from './components/Favicon.vue' import Notifications from './components/Notifications.vue' +import EditTags from './components/EditTags.vue' import { RouterView } from 'vue-router' import { mailbox } from "./stores/mailbox" @@ -11,6 +12,7 @@ export default { components: { Favicon, Notifications, + EditTags }, beforeMount() { @@ -41,4 +43,5 @@ export default { + diff --git a/server/ui-src/components/EditTags.vue b/server/ui-src/components/EditTags.vue new file mode 100644 index 000000000..4824dc030 --- /dev/null +++ b/server/ui-src/components/EditTags.vue @@ -0,0 +1,119 @@ + + + diff --git a/server/ui-src/components/NavTags.vue b/server/ui-src/components/NavTags.vue index 89122d5f5..94b2d091b 100644 --- a/server/ui-src/components/NavTags.vue +++ b/server/ui-src/components/NavTags.vue @@ -86,6 +86,11 @@ export default { Tags
    +
  • + +