Auto update youtube-dl #119 #130

README.md

@@ -29,11 +29,12 @@ any device in podcast client.
 - One-click deployment for AWS.
 - Runs on Windows, Mac OS, Linux, and Docker.
 - Supports ARM.
+- Automatic youtube-dl self update.
 
 ## Dependencies
 
 If you're running the CLI as binary (e.g. not via Docker), you need to make sure that dependencies are available on
-your system. Currently Podsync depends on `youtube-dl` and `ffmpeg`.
+your system. Currently, Podsync depends on `youtube-dl` and `ffmpeg`.
 
 On Mac you can install those with `brew`:
 ```
@@ -78,6 +79,9 @@ vimeo = "{VIMEO_API_TOKEN}"
 [database]
   badger = { truncate = true, file_io = true } # See https://github.com/dgraph-io/badger#memory-usage
 
+[downloader]
+self_update = true # Optional, auto update youtube-dl every 24 hours
+
 # Optional log config. If not specified logs to the stdout
 [log]
 filename = "podsync.log"

cmd/podsync/main.go

@@ -97,7 +97,7 @@ func main() {
 		"date":    date,
 	}).Info("running podsync")
 
-	downloader, err := ytdl.New(ctx)
+	downloader, err := ytdl.New(ctx, cfg.Downloader.SelfUpdate)
 	if err != nil {
 		log.WithError(err).Fatal("youtube-dl error")
 	}

pkg/config/config.go

@@ -101,6 +101,12 @@ type Log struct {
 	Compress bool `toml:"compress"`
 }
 
+// Downloader is a youtube-dl related configuration
+type Downloader struct {
+	// SelfUpdate toggles self update every 24 hour
+	SelfUpdate bool `toml:"self_update"`
+}
+
 type Config struct {
 	// Server is the web server configuration
 	Server Server `toml:"server"`
@@ -113,6 +119,8 @@ type Config struct {
 	Feeds map[string]*Feed
 	// Tokens is API keys to use to access YouTube/Vimeo APIs.
 	Tokens Tokens `toml:"tokens"`
+	// Downloader (youtube-dl) configuration
+	Downloader Downloader `toml:"downloader"`
 }
 
 // LoadConfig loads TOML configuration from a file path

pkg/config/config_test.go

@@ -25,6 +25,9 @@ data_dir = "test/data/"
 [database]
 dir = "/home/user/db/"
 
+[downloader]
+self_update = true
+
 [feeds]
   [feeds.XYZ]
   url = "https://youtube.com/watch?v=ygIUF678y40"
@@ -68,6 +71,8 @@ dir = "/home/user/db/"
 	assert.EqualValues(t, "en", feed.Custom.Language)
 
 	assert.Nil(t, config.Database.Badger)
+
+	assert.True(t, config.Downloader.SelfUpdate)
 }
 
 func TestApplyDefaults(t *testing.T) {

pkg/ytdl/temp_file.go

@@ -0,0 +1,21 @@
+package ytdl
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type tempFile struct {
+	*os.File
+	dir string
+}
+
+func (f *tempFile) Close() error {
+	err := f.File.Close()
+	err1 := os.RemoveAll(f.dir)
+	if err1 != nil {
+		log.Errorf("could not remove temp dir: %v", err1)
+	}
+	return err
+}

pkg/ytdl/ytdl.go

@@ -10,6 +10,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -19,22 +20,21 @@ import (
 	"github.com/mxpv/podsync/pkg/model"
 )
 
-const DownloadTimeout = 10 * time.Minute
+const (
+	DownloadTimeout = 10 * time.Minute
+	UpdatePeriod    = 24 * time.Hour
+)
 
 var (
 	ErrTooManyRequests = errors.New(http.StatusText(http.StatusTooManyRequests))
 )
 
 type YoutubeDl struct {
-	path string
-}
-
-type tempFile struct {
-	*os.File
-	dir string
+	path       string
+	updateLock sync.Mutex // Don't call youtube-dl while self updating
 }
 
-func New(ctx context.Context) (*YoutubeDl, error) {
+func New(ctx context.Context, update bool) (*YoutubeDl, error) {
 	path, err := exec.LookPath("youtube-dl")
 	if err != nil {
 		return nil, errors.Wrap(err, "youtube-dl binary not found")
@@ -58,10 +58,25 @@ func New(ctx context.Context) (*YoutubeDl, error) {
 		return nil, err
 	}
 
+	if update {
+		// Do initial update at launch
+		if err := ytdl.Update(ctx); err != nil {
+			log.WithError(err).Error("failed to update youtube-dl")
+		}
+
+		go func() {
+			for range time.After(UpdatePeriod) {
+				if err := ytdl.Update(context.Background()); err != nil {
+					log.WithError(err).Error("update failed")
+				}
+			}
+		}()
+	}
+
 	return ytdl, nil
 }
 
-func (dl YoutubeDl) ensureDependencies(ctx context.Context) error {
+func (dl *YoutubeDl) ensureDependencies(ctx context.Context) error {
 	found := false
 
 	if path, err := exec.LookPath("ffmpeg"); err == nil {
@@ -93,7 +108,22 @@ func (dl YoutubeDl) ensureDependencies(ctx context.Context) error {
 	return nil
 }
 
-func (dl YoutubeDl) Download(ctx context.Context, feedConfig *config.Feed, episode *model.Episode) (r io.ReadCloser, err error) {
+func (dl *YoutubeDl) Update(ctx context.Context) error {
+	dl.updateLock.Lock()
+	defer dl.updateLock.Unlock()
+
+	log.Info("updating youtube-dl")
+	output, err := dl.exec(ctx, "--update", "--verbose")
+	if err != nil {
+		log.WithError(err).Error(output)
+		return errors.Wrap(err, "failed to self update youtube-dl")
+	}
+
+	log.Info(output)
+	return nil
+}
+
+func (dl *YoutubeDl) Download(ctx context.Context, feedConfig *config.Feed, episode *model.Episode) (r io.ReadCloser, err error) {
 	tmpDir, err := ioutil.TempDir("", "podsync-")
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get temp dir for download")
@@ -112,6 +142,10 @@ func (dl YoutubeDl) Download(ctx context.Context, feedConfig *config.Feed, episo
 	filePath := filepath.Join(tmpDir, fmt.Sprintf("%s.%s", episode.ID, "%(ext)s"))
 
 	args := buildArgs(feedConfig, episode, filePath)
+
+	dl.updateLock.Lock()
+	defer dl.updateLock.Unlock()
+
 	output, err := dl.exec(ctx, args...)
 	if err != nil {
 		log.WithError(err).Errorf("youtube-dl error: %s", filePath)
@@ -140,7 +174,7 @@ func (dl YoutubeDl) Download(ctx context.Context, feedConfig *config.Feed, episo
 	return &tempFile{File: f, dir: tmpDir}, nil
 }
 
-func (dl YoutubeDl) exec(ctx context.Context, args ...string) (string, error) {
+func (dl *YoutubeDl) exec(ctx context.Context, args ...string) (string, error) {
 	ctx, cancel := context.WithTimeout(ctx, DownloadTimeout)
 	defer cancel()
 
@@ -181,12 +215,3 @@ func buildArgs(feedConfig *config.Feed, episode *model.Episode, outputFilePath s
 	args = append(args, "--output", outputFilePath, episode.VideoURL)
 	return args
 }
-
-func (f *tempFile) Close() error {
-	err := f.File.Close()
-	err1 := os.RemoveAll(f.dir)
-	if err1 != nil {
-		log.Errorf("could not remove temp dir: %v", err1)
-	}
-	return err
-}