Merge pull request #132 from mxpv/key-rotation

Support multiple API keys

README.md

@@ -30,6 +30,7 @@ any device in podcast client.
 - Runs on Windows, Mac OS, Linux, and Docker.
 - Supports ARM.
 - Automatic youtube-dl self update.
+- Supports API keys rotation.
 
 ## Dependencies
 
@@ -58,9 +59,13 @@ Here is an example how configuration might look like:
 port = 8080
 data_dir = "/app/data" # Don't change if you run podsync via docker
 
+# Tokens from `Access tokens` section
 [tokens]
-youtube = "{YOUTUBE_API_TOKEN}" # Tokens from `Access tokens` section
-vimeo = "{VIMEO_API_TOKEN}"
+youtube = "YOUTUBE_API_TOKEN" # YouTube API Key. See https://developers.google.com/youtube/registering_an_application
+vimeo = [ # Multiple keys will be rotated.
+  "VIMEO_API_KEY_1", # Vimeo developer keys. See https://developer.vimeo.com/api/guides/start#generate-access-token
+  "VIMEO_API_KEY_2"
+]
 
 [feeds]
   [feeds.ID1]

cmd/podsync/updater.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/mxpv/podsync/pkg/builder"
 	"github.com/mxpv/podsync/pkg/config"
 	"github.com/mxpv/podsync/pkg/db"
 	"github.com/mxpv/podsync/pkg/feed"
@@ -31,14 +32,26 @@ type Updater struct {
 	downloader Downloader
 	db         db.Storage
 	fs         fs.Storage
+	keys       map[model.Provider]feed.KeyProvider
 }
 
 func NewUpdater(config *config.Config, downloader Downloader, db db.Storage, fs fs.Storage) (*Updater, error) {
+	keys := map[model.Provider]feed.KeyProvider{}
+
+	for name, list := range config.Tokens {
+		provider, err := feed.NewKeyProvider(list)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create key provider for %q", name)
+		}
+		keys[name] = provider
+	}
+
 	return &Updater{
 		config:     config,
 		downloader: downloader,
 		db:         db,
 		fs:         fs,
+		keys:       keys,
 	}, nil
 }
 
@@ -78,8 +91,18 @@ func (u *Updater) Update(ctx context.Context, feedConfig *config.Feed) error {
 
 // updateFeed pulls API for new episodes and saves them to database
 func (u *Updater) updateFeed(ctx context.Context, feedConfig *config.Feed) error {
+	info, err := builder.ParseURL(feedConfig.URL)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse URL: %s", feedConfig.URL)
+	}
+
+	keyProvider, ok := u.keys[info.Provider]
+	if !ok {
+		return errors.Errorf("key provider %q not loaded", info.Provider)
+	}
+
 	// Create an updater for this feed type
-	provider, err := feed.New(ctx, feedConfig, u.config.Tokens)
+	provider, err := builder.New(ctx, info.Provider, keyProvider.Get())
 	if err != nil {
 		return err
 	}

go.mod

@@ -2,13 +2,15 @@ module github.com/mxpv/podsync
 
 require (
 	github.com/BrianHicks/finch v0.0.0-20140409222414-419bd73c29ec
-	github.com/BurntSushi/toml v0.3.1
 	github.com/dgraph-io/badger v1.6.0
 	github.com/eduncan911/podcast v1.4.2
 	github.com/gilliek/go-opml v1.0.0
 	github.com/golang/mock v1.4.3
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/jessevdk/go-flags v1.4.0
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/naoina/toml v0.1.1
 	github.com/pkg/errors v0.9.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/silentsokolov/go-vimeo v0.0.0-20190116124215-06829264260c

go.sum

@@ -37,9 +37,15 @@ github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGAR
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
+github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

pkg/builder/builder.go

@@ -0,0 +1,25 @@
+package builder
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	"github.com/mxpv/podsync/pkg/config"
+	"github.com/mxpv/podsync/pkg/model"
+)
+
+type Builder interface {
+	Build(ctx context.Context, cfg *config.Feed) (*model.Feed, error)
+}
+
+func New(ctx context.Context, provider model.Provider, key string) (Builder, error) {
+	switch provider {
+	case model.ProviderYoutube:
+		return NewYouTubeBuilder(key)
+	case model.ProviderVimeo:
+		return NewVimeoBuilder(ctx, key)
+	default:
+		return nil, errors.Errorf("unsupported provider %q", provider)
+	}
+}

pkg/feed/url.go → pkg/builder/url.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"net/url"

pkg/feed/url_test.go → pkg/builder/url_test.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"net/url"

pkg/feed/vimeo.go → pkg/builder/vimeo.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"net/http"

pkg/feed/vimeo_test.go → pkg/builder/vimeo_test.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"context"

pkg/feed/youtube.go → pkg/builder/youtube.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"context"

pkg/feed/youtube_test.go → pkg/builder/youtube_test.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"context"

pkg/config/config.go

@@ -2,11 +2,11 @@ package config
 
 import (
 	"fmt"
+	"io/ioutil"
 	"path/filepath"
-	"time"
 
-	"github.com/BurntSushi/toml"
 	"github.com/hashicorp/go-multierror"
+	"github.com/naoina/toml"
 	"github.com/pkg/errors"
 
 	"github.com/mxpv/podsync/pkg/model"
@@ -44,22 +44,18 @@ type Feed struct {
 	OPML bool `toml:"opml"`
 }
 
+type Filters struct {
+	Title string `toml:"title"`
+	// More filters to be added here
+}
+
 type Custom struct {
 	CoverArt string `toml:"cover_art"`
 	Category string `toml:"category"`
 	Explicit bool   `toml:"explicit"`
 	Language string `toml:"lang"`
 }
 
-type Tokens struct {
-	// YouTube API key.
-	// See https://developers.google.com/youtube/registering_an_application
-	YouTube string `toml:"youtube"`
-	// Vimeo developer key.
-	// See https://developer.vimeo.com/api/guides/start#generate-access-token
-	Vimeo string `toml:"vimeo"`
-}
-
 type Server struct {
 	// Hostname to use for download links
 	Hostname string `toml:"hostname"`
@@ -118,17 +114,21 @@ type Config struct {
 	// ID will be used as feed ID in http://podsync.net/{FEED_ID}.xml
 	Feeds map[string]*Feed
 	// Tokens is API keys to use to access YouTube/Vimeo APIs.
-	Tokens Tokens `toml:"tokens"`
+	Tokens map[model.Provider]StringSlice `toml:"tokens"`
 	// Downloader (youtube-dl) configuration
 	Downloader Downloader `toml:"downloader"`
 }
 
 // LoadConfig loads TOML configuration from a file path
 func LoadConfig(path string) (*Config, error) {
-	config := Config{}
-	_, err := toml.DecodeFile(path, &config)
+	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to load config file")
+		return nil, errors.Wrapf(err, "failed to read config file: %s", path)
+	}
+
+	config := Config{}
+	if err := toml.Unmarshal(data, &config); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal toml")
 	}
 
 	for id, feed := range config.Feeds {
@@ -207,18 +207,3 @@ func (c *Config) applyDefaults(configPath string) {
 		}
 	}
 }
-
-type Duration struct {
-	time.Duration
-}
-
-func (d *Duration) UnmarshalText(text []byte) error {
-	var err error
-	d.Duration, err = time.ParseDuration(string(text))
-	return err
-}
-
-type Filters struct {
-	Title string `toml:"title"`
-	// More filters to be added here
-}

pkg/config/config_test.go

@@ -16,7 +16,10 @@ func TestLoadConfig(t *testing.T) {
 	const file = `
 [tokens]
 youtube = "123"
-vimeo = "321"
+vimeo = [
+  "321",
+  "456"
+]
 
 [server]
 port = 80
@@ -44,15 +47,18 @@ self_update = true
 
 	config, err := LoadConfig(path)
 	assert.NoError(t, err)
-	assert.NotNil(t, config)
+	require.NotNil(t, config)
 
 	assert.Equal(t, "test/data/", config.Server.DataDir)
 	assert.EqualValues(t, 80, config.Server.Port)
 
 	assert.Equal(t, "/home/user/db/", config.Database.Dir)
 
-	assert.Equal(t, "123", config.Tokens.YouTube)
-	assert.Equal(t, "321", config.Tokens.Vimeo)
+	require.Len(t, config.Tokens["youtube"], 1)
+	assert.Equal(t, "123", config.Tokens["youtube"][0])
+	require.Len(t, config.Tokens["vimeo"], 2)
+	assert.Equal(t, "321", config.Tokens["vimeo"][0])
+	assert.Equal(t, "456", config.Tokens["vimeo"][1])
 
 	assert.Len(t, config.Feeds, 1)
 	feed, ok := config.Feeds["XYZ"]
@@ -75,6 +81,28 @@ self_update = true
 	assert.True(t, config.Downloader.SelfUpdate)
 }
 
+func TestLoadEmptyKeyList(t *testing.T) {
+	const file = `
+[tokens]
+vimeo = []
+
+[server]
+data_dir = "/data"
+[feeds]
+  [feeds.A]
+  url = "https://youtube.com/watch?v=ygIUF678y40"
+`
+	path := setup(t, file)
+	defer os.Remove(path)
+
+	config, err := LoadConfig(path)
+	assert.NoError(t, err)
+	require.NotNil(t, config)
+
+	require.Len(t, config.Tokens, 1)
+	require.Len(t, config.Tokens["vimeo"], 0)
+}
+
 func TestApplyDefaults(t *testing.T) {
 	const file = `
 [server]

pkg/config/toml.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+type Duration struct {
+	time.Duration
+}
+
+func (d *Duration) UnmarshalText(text []byte) error {
+	res, err := time.ParseDuration(string(text))
+	if err != nil {
+		return err
+	}
+
+	*d = Duration{res}
+	return nil
+}
+
+// StringSlice is a toml extension that lets you to specify either a string
+// value (a slice with just one element) or a string slice.
+type StringSlice []string
+
+func (s *StringSlice) UnmarshalTOML(decode func(interface{}) error) error {
+	var single string
+	if err := decode(&single); err == nil {
+		*s = []string{single}
+		return nil
+	}
+
+	var slice []string
+	if err := decode(&slice); err == nil {
+		*s = slice
+		return nil
+	}
+
+	return errors.New("failed to decode string (slice) field")
+}