aetaric
diff --git a/‎.gitignore
+1 b/‎.gitignore
+1
diff --git a/‎README.md
+21-18 b/‎README.md
+21-18
diff --git a/‎check/checkrr.go
+103-64 b/‎check/checkrr.go
+103-64
@@ -6,5 +6,6 @@ test/
 checkrr
 .vscode
 badfiles.csv
+checkrr.log
 webserver/build
 webserver/node_modules
@@ -9,20 +9,29 @@ I've been running a media library for the past ~ 8 years migrating my library be
 Checkrr runs various checks (ffprobe, magic number, mimetype, and file hash on subsequent runs to drastically improve speed) on the path you specify as `checkpath` in the config. 
 
 * If the file passes inspection, the hash is recorded in a bbolt flatfile DB so future runs are insanely fast on large libraries. 
-* If the file fails all checks checkrr will check sonarr and/or radarr for the file removing it and requesting a new version via the correct system (assuming they are enabled... you could just run checkrr in a no-op state by setting `sonarr.process: false` and `radarr.process: false` in the config and then egrep the output like so `checkrr check | egrep "Hash Mismatch|not a recongized file type"` for environments that do not run either of these.)
+* If the file fails all checks checkrr will check sonarr and/or radarr for the file removing it and requesting a new version via the correct system (assuming they are enabled... you could just run checkrr in a no-op state by setting `sonarr.process: false` and `radarr.process: false` in the config and then egrep the output like so `checkrr check | egrep "Hash Mismatch|not a recognized file type"` for environments that do not run either of these.)
 
 ## Screenshots
 ![Idle screenshot](./screenshots/Idle.png?raw=true)
 ![Running screenshot](./screenshots/Running.png?raw=true)
 
-## Installation
-cli:
-Grab a release from the releases page.
+## Installation and running checkrr
+### cli
+* Install prerequisite packages via your package manager or by downloading the installer (for windows): ffmpeg
+* Make sure ffprobe is in your $PATH var. If you installed from a Linux/macOS package manager, it is. If you are on windows, you'll need to make sure you can run ffprobe from a basic command prompt/powershell.
+* Grab a release from the releases page.
+* Copy the example config from the repo: `wget https://raw.githubusercontent.com/aetaric/checkrr/main/checkrr.yaml.example -O checkrr.yaml`
+* Edit the config in your favorite editor. Make sure you remove any sections you aren't using. (If you aren't using influxdb 1 and/or 2 for example, you should remove the entire stats block from your config.)If you aren't sure what the minimal config file can look like, check https://raw.githubusercontent.com/aetaric/checkrr/main/checkrr.yaml.minimal. 
+* To run checkrr as a daemon, use `checkrr -c /path/to/checkrr.yaml`. If you'd like checkrr to run once and then exit (useful for running in your own cron daemon) `checkrr -c /path/to/checkrr.yaml --run-once`.
 
-docker:
-`docker pull ghcr.io/aetaric/checkrr:latest`
+### docker
+YOU MUST CREATE THE CONFIG AND DB FILES BEFORE STARTING. checkrr will complain if these are directories. Docker doesn't know you want to mount a file unless it already exists.
+
+* creating empty db file: `touch checkrr.db`
+* creating a config file from the example: `wget https://raw.githubusercontent.com/aetaric/checkrr/main/checkrr.yaml.example -O checkrr.yaml`
+_make sure you edit the example config from the defaults. Remove any unused sections._
+While editing the example you might want to add path mappings if the path to your media is differs from arr services and checkrr.
 
-## Usage
 
 ### Running Checkrr
 cli as a daemon:
@@ -52,20 +61,14 @@ services:
     restart: on-failure
 ```
 
-## Upgrading to 2.x
-Checkrr 2.x has a more organized config file and quite a reduction in CLI flags. Checkout `checkrr --help` for the flag changes. You will have to manually conform your config file to the example file in the repo; checkrr no longer outputs a default config.
-
-## Unknown file deletion
-If you are feeling especially spicy, there is `RemoveUnknownFiles` flag in the config. This flag is destructive. It will remove any file that isn't detected as a valid Video, Audio, Document, or plain text file. 
-
-**Seriously** I don't recommend you run this on the first pass if at all. You are very likely to lose something you didn't expect to lose. 
-
-Before using this flag, run checkrr and read the full output to ensure you don't nuke a file that you don't want to lose. Run it again with sonarr and/or radarr enabled.
+### unRAID using mrslaw's community applications repo
+Please note the Additional Requirements on the details screen prior to pressing install. mrslaw has all the commands you need to run there.
 
-*I am not responsible for your use of this flag. I will not help you sort out any damage you cause to your library. Issues opened around this flag's usage will be summarily closed as PEBCAK.*
+## Upgrading to 3.1 or newer
+checkrr > 3.1 has changed the way arr services are handled. Please review the example config and bring your config into compliance prior to running checkrr. With the 3.1 release checkrr supports having multiple of each arr service. So you could have 3 sonarr instances connected. Each arr config under `arr:` has a `service` key to tell checkrr what service type it is. This can be set to `sonarr`, `radarr`, or `lidarr`. Please note that if you are running on docker, you will likely want to setup path mappings for each service. checkrr will attempt to translate the paths that the arr services see when working with their APIs.
 
 ## Building
-Should you want to build checkrr, you can do so with the following:
+Should you want to build checkrr from source, you can do so with the following:
 `cd webserver && yarn build && cd .. && go build`
 Please note, if you build checkrr yourself, you will be told to download the official release if you open an issue for a bug.
 
 
@@ -8,12 +8,14 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/aetaric/checkrr/connections"
 	"github.com/aetaric/checkrr/features"
 	"github.com/aetaric/checkrr/hidden"
 	"github.com/aetaric/checkrr/notifications"
 	"github.com/h2non/filetype"
+	"github.com/h2non/filetype/matchers"
 	"github.com/kalafut/imohash"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
@@ -27,10 +29,11 @@ type Checkrr struct {
 	Running       bool
 	csv           features.CSV
 	notifications notifications.Notifications
-	sonarr        connections.Sonarr
-	radarr        connections.Radarr
-	lidarr        connections.Lidarr
+	sonarr        []connections.Sonarr
+	radarr        []connections.Radarr
+	lidarr        []connections.Lidarr
 	ignoreExts    []string
+	ignorePaths   []string
 	ignoreHidden  bool
 	config        *viper.Viper
 	FullConfig    *viper.Viper
@@ -57,11 +60,6 @@ func (c *Checkrr) Run() {
 	// Connect to Sonarr, Radarr, and Lidarr
 	c.connectServices()
 
-	// Unknown File deletion
-	if c.config.GetBool("removeunknownfiles") {
-		log.WithFields(log.Fields{"startup": true, "unknownFiles": "enabled"}).Warn(`unknown file deletion is on. You may lose files that are not tracked by services you've enabled in the config. This will still delete files even if those integrations are disabled.`)
-	}
-
 	// Connect to notifications
 	c.connectNotifications()
 
@@ -72,8 +70,16 @@ func (c *Checkrr) Run() {
 	}
 
 	c.ignoreExts = c.config.GetStringSlice("ignoreexts")
+	c.ignorePaths = c.config.GetStringSlice("ignorepaths")
 	c.ignoreHidden = c.config.GetBool("ignorehidden")
 
+	// I'm tired of waiting for filetype to support this. We'll force it by adding to the matchers on the fly.
+	// TODO: if h2non/filetype#120 ever gets completed, remove this logic
+	ts := filetype.AddType("ts", "MPEG-TS")
+	m2ts := filetype.AddType("m2ts", "MPEG-TS")
+	matchers.Video[ts] = mpegts_matcher
+	matchers.Video[m2ts] = mpegts_matcher
+
 	c.Stats.Start()
 
 	log.Debug(c.config.GetStringSlice("checkpath"))
@@ -98,7 +104,17 @@ func (c *Checkrr) Run() {
 
 				if c.ignoreHidden {
 					i, _ := hidden.IsHidden(path)
-					ignore = i
+					if !ignore {
+						ignore = i
+					}
+				}
+
+				for _, v := range c.ignorePaths {
+					if strings.Contains(path, v) {
+						if !ignore {
+							ignore = true
+						}
+					}
 				}
 
 				if !ignore {
@@ -161,31 +177,45 @@ func (c *Checkrr) FromConfig(conf *viper.Viper) {
 }
 
 func (c *Checkrr) connectServices() {
-	if viper.GetViper().Sub("sonarr") != nil {
-		c.sonarr = connections.Sonarr{}
-		c.sonarr.FromConfig(*viper.GetViper().Sub("sonarr"))
-		sonarrConnected, sonarrMessage := c.sonarr.Connect()
-		log.WithFields(log.Fields{"Startup": true, "Sonarr Connected": sonarrConnected}).Info(sonarrMessage)
-	} else {
-		log.WithFields(log.Fields{"Startup": true, "Sonarr Connected": false}).Info("Sonarr integration not enabled. Files will not be fixed. (if you expected a no-op, this is fine)")
-	}
+	if viper.GetViper().GetStringMap("arr") != nil {
+		arrConfig := viper.GetViper().Sub("arr")
+		arrKeys := viper.GetViper().Sub("arr").AllKeys()
+		for _, key := range arrKeys {
+			if strings.Contains(key, "service") {
+				k := strings.Split(key, ".")[0]
+				config := arrConfig.Sub(k)
+
+				if config.GetString("service") == "sonarr" {
+					sonarr := connections.Sonarr{}
+					sonarr.FromConfig(config)
+					sonarrConnected, sonarrMessage := sonarr.Connect()
+					log.WithFields(log.Fields{"Startup": true, fmt.Sprintf("Sonarr \"%s\" Connected", k): sonarrConnected}).Info(sonarrMessage)
+					if sonarrConnected {
+						c.sonarr = append(c.sonarr, sonarr)
+					}
+				}
 
-	if viper.GetViper().Sub("radarr") != nil {
-		c.radarr = connections.Radarr{}
-		c.radarr.FromConfig(*viper.GetViper().Sub("radarr"))
-		radarrConnected, radarrMessage := c.radarr.Connect()
-		log.WithFields(log.Fields{"Startup": true, "Radarr Connected": radarrConnected}).Info(radarrMessage)
-	} else {
-		log.WithFields(log.Fields{"Startup": true, "Radarr Connected": false}).Info("Radarr integration not enabled. Files will not be fixed. (if you expected a no-op, this is fine)")
-	}
+				if config.GetString("service") == "radarr" {
+					radarr := connections.Radarr{}
+					radarr.FromConfig(config)
+					radarrConnected, radarrMessage := radarr.Connect()
+					log.WithFields(log.Fields{"Startup": true, fmt.Sprintf("Radarr \"%s\" Connected", k): radarrConnected}).Info(radarrMessage)
+					if radarrConnected {
+						c.radarr = append(c.radarr, radarr)
+					}
+				}
 
-	if viper.GetViper().Sub("lidarr") != nil {
-		c.lidarr = connections.Lidarr{}
-		c.lidarr.FromConfig(*viper.GetViper().Sub("lidarr"))
-		lidarrConnected, lidarrMessage := c.lidarr.Connect()
-		log.WithFields(log.Fields{"Startup": true, "Lidarr Connected": lidarrConnected}).Info(lidarrMessage)
-	} else {
-		log.WithFields(log.Fields{"Startup": true, "Lidarr Connected": false}).Info("Lidarr integration not enabled. Files will not be fixed. (if you expected a no-op, this is fine)")
+				if config.GetString("service") == "lidarr" {
+					lidarr := connections.Lidarr{}
+					lidarr.FromConfig(config)
+					lidarrConnected, lidarrMessage := lidarr.Connect()
+					log.WithFields(log.Fields{"Startup": true, fmt.Sprintf("Lidarr \"%s\" Connected", k): lidarrConnected}).Info(lidarrMessage)
+					if lidarrConnected {
+						c.lidarr = append(c.lidarr, lidarr)
+					}
+				}
+			}
+		}
 	}
 }
 
@@ -257,7 +287,7 @@ func (c *Checkrr) checkFile(path string) {
 		content := http.DetectContentType(buf)
 		log.WithFields(log.Fields{"FFProbe": false, "Type": "Unknown"}).Debugf("File \"%v\" is of type \"%v\"", path, content)
 		buf = nil
-		log.WithFields(log.Fields{"FFProbe": false, "Type": "Unknown"}).Infof("File \"%v\" is not a recongized file type", path)
+		log.WithFields(log.Fields{"FFProbe": false, "Type": "Unknown"}).Infof("File \"%v\" is not a recognized file type", path)
 		c.notifications.Notify("Unknown file detected", fmt.Sprintf("\"%v\" is not a Video, Audio, Image, Subtitle, or Plaintext file.", path), "unknowndetected", path)
 		c.Stats.UnknownFileCount++
 		c.Stats.Write("UnknownFiles", c.Stats.UnknownFileCount)
@@ -267,40 +297,38 @@ func (c *Checkrr) checkFile(path string) {
 }
 
 func (c *Checkrr) deleteFile(path string) {
-	if c.sonarr.Process && c.sonarr.MatchPath(path) {
-		c.sonarr.RemoveFile(path)
-		c.notifications.Notify("File Reacquire", fmt.Sprintf("\"%v\" was sent to sonarr to be reacquired", path), "reacquire", path)
-		c.Stats.SonarrSubmissions++
-		c.Stats.Write("Sonarr", c.Stats.SonarrSubmissions)
-		c.recordBadFile(path, "sonarr")
-	} else if c.radarr.Process && c.radarr.MatchPath(path) {
-		c.radarr.RemoveFile(path)
-		c.notifications.Notify("File Reacquire", fmt.Sprintf("\"%v\" was sent to radarr to be reacquired", path), "reacquire", path)
-		c.Stats.RadarrSubmissions++
-		c.Stats.Write("Radarr", c.Stats.RadarrSubmissions)
-		c.recordBadFile(path, "radarr")
-	} else if c.lidarr.Process && c.lidarr.MatchPath(path) {
-		c.lidarr.RemoveFile(path)
-		c.notifications.Notify("File Reacquire", fmt.Sprintf("\"%v\" was sent to lidarr to be reacquired", path), "reacquire", path)
-		c.Stats.LidarrSubmissions++
-		c.Stats.Write("Lidarr", c.Stats.LidarrSubmissions)
-		c.recordBadFile(path, "lidarr")
-	} else {
-		log.WithFields(log.Fields{"Unknown File": true}).Infof("Couldn't find a target for file \"%v\". File is unknown.", path)
-		c.recordBadFile(path, "unknown")
-		if c.config.GetBool("removeunknownfiles") {
-			e := os.Remove(path)
-			if e != nil {
-				log.WithFields(log.Fields{"FFProbe": false, "Type": "Unknown", "Deleted": false}).Warnf("Could not delete File: \"%v\"", path)
-				return
-			}
-			log.WithFields(log.Fields{"FFProbe": false, "Type": "Unknown", "Deleted": true}).Warnf("Removed File: \"%v\"", path)
-			c.notifications.Notify("Unknown file deleted", fmt.Sprintf("\"%v\" was removed.", path), "unknowndeleted", path)
-			c.Stats.UnknownFilesDeleted++
-			c.Stats.Write("UnknownDelete", c.Stats.UnknownFilesDeleted)
+	for _, sonarr := range c.sonarr {
+		if sonarr.Process && sonarr.MatchPath(path) {
+			sonarr.RemoveFile(path)
+			c.notifications.Notify("File Reacquire", fmt.Sprintf("\"%v\" was sent to sonarr to be reacquired", path), "reacquire", path)
+			c.Stats.SonarrSubmissions++
+			c.Stats.Write("Sonarr", c.Stats.SonarrSubmissions)
+			c.recordBadFile(path, "sonarr")
 			return
 		}
 	}
+	for _, radarr := range c.radarr {
+		if radarr.Process && radarr.MatchPath(path) {
+			radarr.RemoveFile(path)
+			c.notifications.Notify("File Reacquire", fmt.Sprintf("\"%v\" was sent to radarr to be reacquired", path), "reacquire", path)
+			c.Stats.RadarrSubmissions++
+			c.Stats.Write("Radarr", c.Stats.RadarrSubmissions)
+			c.recordBadFile(path, "radarr")
+			return
+		}
+	}
+	for _, lidarr := range c.lidarr {
+		if lidarr.Process && lidarr.MatchPath(path) {
+			lidarr.RemoveFile(path)
+			c.notifications.Notify("File Reacquire", fmt.Sprintf("\"%v\" was sent to lidarr to be reacquired", path), "reacquire", path)
+			c.Stats.LidarrSubmissions++
+			c.Stats.Write("Lidarr", c.Stats.LidarrSubmissions)
+			c.recordBadFile(path, "lidarr")
+			return
+		}
+	}
+	log.WithFields(log.Fields{"Unknown File": true}).Infof("Couldn't find a target for file \"%v\". File is unknown.", path)
+	c.recordBadFile(path, "unknown")
 }
 
 func (c *Checkrr) recordBadFile(path string, fileType string) {
@@ -325,13 +353,24 @@ func (c *Checkrr) recordBadFile(path string, fileType string) {
 			return nil
 		}
 	})
+
 	if err != nil {
 		log.WithFields(log.Fields{"DB Update": "Failure"}).Warnf("Error: %v", err.Error())
 	}
+
+	if c.config.GetString("csvfile") != "" {
+		c.csv.Write(path, fileType)
+	}
 }
 
 type BadFile struct {
 	FileExt   string `json:"fileExt"`
 	Reacquire bool   `json:"reacquire"`
 	Service   string `json:"service"`
 }
+
+// TODO: if h2non/filetype#120 ever gets completed, remove this logic
+func mpegts_matcher(buf []byte) bool {
+	return len(buf) > 1 &&
+		buf[0] == 0x47
+}