Optimized builds, logging and app config

frontend/src/locales/de.yaml

@@ -132,6 +132,7 @@ tags:
   sys:file:location:dropbox: Dropbox
   sys:file:location:onedrive: OneDrive
   sys:file:location:google-drive: Google Drive
+  sys:file:location:pcloud: pCloud
   sys:file:location:elsewhere: Sonstwo
 
   sys:file:btime-year: Erstelltes Jahr

frontend/src/locales/en.yaml

@@ -132,6 +132,7 @@ tags:
   sys:file:location:dropbox: Dropbox
   sys:file:location:onedrive: OneDrive
   sys:file:location:google-drive: Google Drive
+  sys:file:location:pcloud: pCloud
   sys:file:location:elsewhere: Elsewhere
 
   sys:file:btime-year: Creation year

service/.gitignore

@@ -1,2 +1,3 @@
 /internal/webservice/.frontend/
-/ablegram
+/Ablegram*
+/*.log

service/FyneApp.toml

@@ -1,8 +1,8 @@
 Website = "https://www.ablegram.app"
 
 [Details]
-Icon = "assets/icon.png"
-Name = "Ablegram"
-ID = "com.ablegram.app"
-Version = "1.0.0"
-Build = 1
+  Icon = "assets/icon.png"
+  Name = "Ablegram"
+  ID = "com.ablegram.app"
+  Version = "1.0.0"
+  Build = 5

service/PACKAGING.md

@@ -0,0 +1,18 @@
+# Packaging
+
+## Setup
+
+Based on the following documents:
+
+- https://github.com/fyne-io/fyne-cross
+
+```shell
+sudo su
+wget https://go.dev/dl/go1.21.3.src.tar.gz
+tar xvfz go1.21.3.src.tar.gz
+mv go /usr/local/
+rm rm go1.21.3.src.tar.gz
+
+go install github.com/fyne-io/fyne-cross@latest
+fyne-cross linux --pull
+```

service/flags.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"flag"
+	"github.com/adrianrudnik/ablegram/internal/config"
+)
+
+func parseFlags(c *config.Config) {
+	logLevel := flag.String("log-level", "info", "Set the log level [debug, info]")
+	logToFiles := flag.Bool("enable-logs", false, "Enable debug log writing to files")
+	logScannedFolders := flag.Bool("enable-scanned-log", false, "Enable scanned paths log file")
+
+	noBrowserFlag := flag.Bool("no-browser", false, "Skip the automatic browser opening")
+	noGuiFlag := flag.Bool("no-gui", false, "Do no start the GUI.")
+	noWebserviceFlag := flag.Bool("no-webservice", false, "Do no start the webservice")
+
+	flag.Parse()
+
+	c.Log.Level = *logLevel
+	c.Log.ToFiles = *logToFiles
+	c.Log.ScannedFolders = *logScannedFolders
+
+	c.Behaviour.BrowserAutostart = !*noBrowserFlag
+	c.Behaviour.ShowGui = !*noGuiFlag
+	c.Behaviour.WebserviceAutostart = !*noWebserviceFlag
+}

service/internal/collector/collector.go

@@ -1,6 +1,7 @@
 package collector
 
 import (
+	"github.com/adrianrudnik/ablegram/internal/config"
 	"github.com/adrianrudnik/ablegram/internal/pipeline"
 	"github.com/adrianrudnik/ablegram/internal/pusher"
 	"github.com/rs/zerolog"
@@ -21,18 +22,17 @@ var excludePaths = []string{
 }
 
 var excludeFolders = []string{
-	".git",
-	".idea",
+	"node_modules",
 }
 
 type Collection struct {
 	files []string
 }
 
-func Collect(path string, filesChan chan<- *pipeline.FilesForProcessorMsg, broadcastChan chan<- interface{}) error {
+func Collect(c *config.Config, path string, filesChan chan<- *pipeline.FilesForProcessorMsg, broadcastChan chan<- interface{}) error {
 	allowedExtensions := []string{".als"}
 
-	err := findFilesByExtension(path, allowedExtensions, filesChan, broadcastChan)
+	err := findFilesByExtension(c, path, allowedExtensions, filesChan, broadcastChan)
 	if err != nil {
 		return err
 	}
@@ -44,27 +44,41 @@ func Collect(path string, filesChan chan<- *pipeline.FilesForProcessorMsg, broad
 	return nil
 }
 
-func findFilesByExtension(root string, extensions []string, filesChan chan<- *pipeline.FilesForProcessorMsg, broadcastChan chan<- interface{}) error {
+func findFilesByExtension(c *config.Config, root string, extensions []string, filesChan chan<- *pipeline.FilesForProcessorMsg, broadcastChan chan<- interface{}) error {
+
+	folders := make([]string, 0, 1000000)
+
 	err := filepath.WalkDir(root, func(s string, d fs.DirEntry, e error) error {
+		if e != nil {
+			Logger.Warn().Err(e).Str("path", s).Msg("Skipped folder due to error")
+			return nil
+		}
+
+		// Exclude folders beginning with a dot
+		if c.Collector.ExcludeSystemFolders && d.IsDir() && strings.HasPrefix(d.Name(), ".") {
+			Logger.Debug().Str("path", s).Msg("Skipping dot folder")
+			return filepath.SkipDir
+		}
+
 		// Exclude paths by prefix
-		if d.IsDir() && slices.IndexFunc(excludePaths, func(s string) bool {
+		if c.Collector.ExcludeSystemFolders && d.IsDir() && slices.IndexFunc(excludePaths, func(s string) bool {
 			return strings.HasPrefix(d.Name(), s)
 		}) != -1 {
-			Logger.Info().Str("path", s).Msg("Skipping excluded path")
+			Logger.Debug().Str("path", s).Msg("Skipping excluded path")
 			return filepath.SkipDir
 		}
 
 		// Exclude folders by name
-		if d.IsDir() && slices.IndexFunc(excludeFolders, func(s string) bool {
+		if c.Collector.ExcludeSystemFolders && d.IsDir() && slices.IndexFunc(excludeFolders, func(s string) bool {
 			return s == filepath.Base(d.Name())
 		}) != -1 {
-			Logger.Info().Str("path", s).Msg("Skipping excluded folder")
+			Logger.Debug().Str("path", s).Msg("Skipping excluded folder")
 			return filepath.SkipDir
 		}
 
-		if e != nil {
-			Logger.Warn().Err(e).Str("path", s).Msg("Skipped folder due to error")
-			return nil
+		// Log at least the folders we are visiting
+		if d.IsDir() {
+			folders = append(folders, s)
 		}
 
 		for _, ext := range extensions {
@@ -82,6 +96,15 @@ func findFilesByExtension(root string, extensions []string, filesChan chan<- *pi
 		return nil
 	})
 
+	if c.Log.ScannedFolders {
+		scanLogPath := config.GetRelativeFilePath(".scanned-folders.log")
+		lines := strings.Join([]string(folders), "\n")
+		err := os.WriteFile(scanLogPath, []byte(lines), 0666)
+		if err != nil {
+			Logger.Warn().Err(err).Msg("Failed to write scanned folders to file")
+		}
+	}
+
 	if err != nil {
 		return err
 	}

service/internal/collector/worker.go

@@ -1,6 +1,7 @@
 package collector
 
 import (
+	"github.com/adrianrudnik/ablegram/internal/config"
 	"github.com/adrianrudnik/ablegram/internal/pipeline"
 	"github.com/adrianrudnik/ablegram/internal/stats"
 )
@@ -22,35 +23,35 @@ func NewWorkerPool(workerCount int, filesChan chan<- *pipeline.FilesForProcessor
 	}
 }
 
-func (p *WorkerPool) Run(progress *stats.ProcessProgress, paths []string) {
+func (wp *WorkerPool) Run(c *config.Config, p *stats.ProcessProgress) {
 	Logger.Info().
-		Int("count", p.workerCount).
-		Strs("paths", paths).
+		Int("count", wp.workerCount).
+		Strs("paths", c.Collector.SearchablePaths).
 		Msg("Starting collector workers")
 
 	// Spool up workers first
-	for i := 0; i < p.workerCount; i++ {
-		go p.doWork(progress)
+	for i := 0; i < wp.workerCount; i++ {
+		go wp.doWork(c, p)
 	}
 
 	// Pipe in paths next
-	for _, path := range paths {
-		p.inPathChan <- path
+	for _, path := range c.Collector.SearchablePaths {
+		wp.inPathChan <- path
 	}
 }
 
-func (p *WorkerPool) doWork(progress *stats.ProcessProgress) {
+func (wp *WorkerPool) doWork(c *config.Config, p *stats.ProcessProgress) {
 	for {
 		select {
-		case path := <-p.inPathChan:
-			progress.Add()
+		case path := <-wp.inPathChan:
+			p.Add()
 
-			err := Collect(path, p.outFilesChan, p.pushChan)
+			err := Collect(c, path, wp.outFilesChan, wp.pushChan)
 			if err != nil {
 				Logger.Warn().Err(err).Str("path", path).Msg("Failed to collect files")
 			}
 
-			progress.Done()
+			p.Done()
 		}
 	}
 }