From 67478ac9f4940df09da40cb948136ff85a011426 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 16 Apr 2020 15:14:46 -0700 Subject: [PATCH] Fix key not found error #120 --- cmd/podsync/main.go | 2 +- cmd/podsync/updater.go | 9 ++++----- pkg/db/badger.go | 10 +++++++--- pkg/db/storage.go | 5 ----- pkg/feed/common.go | 5 ----- pkg/feed/opml.go | 36 ++++++++++++++++++++++-------------- pkg/feed/vimeo.go | 6 +++--- pkg/feed/youtube.go | 4 ++-- pkg/model/errors.go | 11 +++++++++++ 9 files changed, 50 insertions(+), 38 deletions(-) create mode 100644 pkg/model/errors.go diff --git a/cmd/podsync/main.go b/cmd/podsync/main.go index 918b00b9..a2fbb98b 100644 --- a/cmd/podsync/main.go +++ b/cmd/podsync/main.go @@ -123,7 +123,7 @@ func main() { if err := updater.Update(ctx, feed); err != nil { log.WithError(err).Errorf("failed to update feed: %s", feed.URL) } else { - log.Infof("Next update of %s: %s", feed.ID, c.Entry(m[feed.ID]).Next) + log.Infof("next update of %s: %s", feed.ID, c.Entry(m[feed.ID]).Next) } case <-ctx.Done(): return ctx.Err() diff --git a/cmd/podsync/updater.go b/cmd/podsync/updater.go index 772cd7c4..db425c7e 100644 --- a/cmd/podsync/updater.go +++ b/cmd/podsync/updater.go @@ -52,19 +52,19 @@ func (u *Updater) Update(ctx context.Context, feedConfig *config.Feed) error { started := time.Now() if err := u.updateFeed(ctx, feedConfig); err != nil { - return err + return errors.Wrap(err, "update failed") } if err := u.downloadEpisodes(ctx, feedConfig); err != nil { - return err + return errors.Wrap(err, "download failed") } if err := u.buildXML(ctx, feedConfig); err != nil { - return err + return errors.Wrap(err, "xml build failed") } if err := u.buildOPML(ctx); err != nil { - return err + return errors.Wrap(err, "opml build failed") } if err := u.cleanup(ctx, feedConfig); err != nil { @@ -263,7 +263,6 @@ func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error { } func (u *Updater) buildOPML(ctx context.Context) error { - // Build OPML with data received from builder log.Debug("building podcast OPML") opml, err := feed.BuildOPML(ctx, u.config, u.db, u.fs) diff --git a/pkg/db/badger.go b/pkg/db/badger.go index fe3c0ddf..efebeddc 100644 --- a/pkg/db/badger.go +++ b/pkg/db/badger.go @@ -60,7 +60,7 @@ func NewBadger(config *config.Database) (*Badger, error) { storage := &Badger{db: db} if err := db.Update(func(txn *badger.Txn) error { - if err := storage.setObj(txn, []byte(versionPath), CurrentVersion, false); err != nil && err != ErrAlreadyExists { + if err := storage.setObj(txn, []byte(versionPath), CurrentVersion, false); err != nil && err != model.ErrAlreadyExists { return err } return nil @@ -100,7 +100,7 @@ func (b *Badger) AddFeed(_ context.Context, feedID string, feed *model.Feed) err for _, episode := range feed.Episodes { episodeKey := b.getKey(episodePath, feedID, episode.ID) err := b.setObj(txn, episodeKey, episode, false) - if err == nil || err == ErrAlreadyExists { + if err == nil || err == model.ErrAlreadyExists { // Do nothing } else { return errors.Wrapf(err, "failed to save episode %q", feedID) @@ -261,7 +261,7 @@ func (b *Badger) setObj(txn *badger.Txn, key []byte, obj interface{}, overwrite // Overwrites are not allowed, make sure there is no object with the given key _, err := txn.Get(key) if err == nil { - return ErrAlreadyExists + return model.ErrAlreadyExists } else if err == badger.ErrKeyNotFound { // Key not found, do nothing } else { @@ -280,6 +280,10 @@ func (b *Badger) setObj(txn *badger.Txn, key []byte, obj interface{}, overwrite func (b *Badger) getObj(txn *badger.Txn, key []byte, out interface{}) error { item, err := txn.Get(key) if err != nil { + if err == badger.ErrKeyNotFound { + return model.ErrNotFound + } + return err } diff --git a/pkg/db/storage.go b/pkg/db/storage.go index 79b80302..3da4dc85 100644 --- a/pkg/db/storage.go +++ b/pkg/db/storage.go @@ -2,7 +2,6 @@ package db import ( "context" - "errors" "github.com/mxpv/podsync/pkg/model" ) @@ -13,10 +12,6 @@ const ( CurrentVersion = 1 ) -var ( - ErrAlreadyExists = errors.New("object already exists") -) - type Storage interface { Close() error Version() (int, error) diff --git a/pkg/feed/common.go b/pkg/feed/common.go index c8823c06..935dbee1 100644 --- a/pkg/feed/common.go +++ b/pkg/feed/common.go @@ -10,11 +10,6 @@ import ( "github.com/mxpv/podsync/pkg/model" ) -var ( - ErrNotFound = errors.New("resource not found") - ErrQuotaExceeded = errors.New("query limit is exceeded") -) - type Builder interface { Build(ctx context.Context, cfg *config.Feed) (*model.Feed, error) } diff --git a/pkg/feed/opml.go b/pkg/feed/opml.go index ac77224b..c98aee00 100644 --- a/pkg/feed/opml.go +++ b/pkg/feed/opml.go @@ -6,8 +6,10 @@ import ( "github.com/gilliek/go-opml/opml" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" "github.com/mxpv/podsync/pkg/config" + "github.com/mxpv/podsync/pkg/model" ) func BuildOPML(ctx context.Context, config *config.Config, db feedProvider, provider urlProvider) (string, error) { @@ -17,25 +19,31 @@ func BuildOPML(ctx context.Context, config *config.Config, db feedProvider, prov for _, feed := range config.Feeds { f, err := db.GetFeed(ctx, feed.ID) - if err != nil { - return "", err + if err == model.ErrNotFound { + // As we update OPML on per-feed basis, some feeds may not yet be populated in database. + log.Debugf("can't find configuration for feed %q, ignoring opml", feed.ID) + continue + } else if err != nil { + return "", errors.Wrapf(err, "failed to query feed %q", feed.ID) } - if feed.OPML { - downloadURL, err := provider.URL(ctx, "", fmt.Sprintf("%s.xml", feed.ID)) - if err != nil { - return "", errors.Wrapf(err, "failed to get feed URL for %q", feed.ID) - } + if !feed.OPML { + continue + } - outline := opml.Outline{ - Title: f.Title, - Text: f.Description, - Type: "rss", - XMLURL: downloadURL, - } + downloadURL, err := provider.URL(ctx, "", fmt.Sprintf("%s.xml", feed.ID)) + if err != nil { + return "", errors.Wrapf(err, "failed to get feed URL for %q", feed.ID) + } - doc.Body.Outlines = append(doc.Body.Outlines, outline) + outline := opml.Outline{ + Title: f.Title, + Text: f.Description, + Type: "rss", + XMLURL: downloadURL, } + + doc.Body.Outlines = append(doc.Body.Outlines, outline) } out, err := doc.XML() diff --git a/pkg/feed/vimeo.go b/pkg/feed/vimeo.go index 2eb651b0..6184cba2 100644 --- a/pkg/feed/vimeo.go +++ b/pkg/feed/vimeo.go @@ -41,7 +41,7 @@ func (v *VimeoBuilder) queryChannel(feed *model.Feed) error { ch, resp, err := v.client.Channels.Get(channelID) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return ErrNotFound + return model.ErrNotFound } return errors.Wrapf(err, "failed to query channel with id %q", channelID) @@ -64,7 +64,7 @@ func (v *VimeoBuilder) queryGroup(feed *model.Feed) error { gr, resp, err := v.client.Groups.Get(groupID) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return ErrNotFound + return model.ErrNotFound } return errors.Wrapf(err, "failed to query group with id %q", groupID) @@ -87,7 +87,7 @@ func (v *VimeoBuilder) queryUser(feed *model.Feed) error { user, resp, err := v.client.Users.Get(userID) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return ErrNotFound + return model.ErrNotFound } return errors.Wrapf(err, "failed to query user with id %q", userID) diff --git a/pkg/feed/youtube.go b/pkg/feed/youtube.go index e1df5484..a927712b 100644 --- a/pkg/feed/youtube.go +++ b/pkg/feed/youtube.go @@ -57,7 +57,7 @@ func (yt *YouTubeBuilder) listChannels(ctx context.Context, linkType link.Type, } if len(resp.Items) == 0 { - return nil, ErrNotFound + return nil, model.ErrNotFound } item := resp.Items[0] @@ -81,7 +81,7 @@ func (yt *YouTubeBuilder) listPlaylists(ctx context.Context, id, channelID strin } if len(resp.Items) == 0 { - return nil, ErrNotFound + return nil, model.ErrNotFound } item := resp.Items[0] diff --git a/pkg/model/errors.go b/pkg/model/errors.go new file mode 100644 index 00000000..2c4fcbeb --- /dev/null +++ b/pkg/model/errors.go @@ -0,0 +1,11 @@ +package model + +import ( + "errors" +) + +var ( + ErrAlreadyExists = errors.New("object already exists") + ErrNotFound = errors.New("not found") + ErrQuotaExceeded = errors.New("query limit is exceeded") +)