feat: add refresh on filetrees and filelists (#30)

* feat: add refresh on filetrees and filelists

Signed-off-by: Thomas Bétrancourt <[email protected]>

configstore.go

@@ -17,8 +17,11 @@ var (
 
 func init() {
 	RegisterProviderFactory("file", fileProvider)
+	RegisterProviderFactory("file+refresh", fileRefreshProvider)
 	RegisterProviderFactory("filelist", fileListProvider)
+	RegisterProviderFactory("filelist+refresh", fileListRefreshProvider)
 	RegisterProviderFactory("filetree", fileTreeProvider)
+	RegisterProviderFactory("filetree+refresh", fileTreeRefreshProvider)
 	RegisterProviderFactory("env", envProvider)
 }
 

defaultstore.go

@@ -74,12 +74,24 @@ func FileTree(dirname string) {
 	DefaultStore.FileTree(dirname)
 }
 
+// FileTreeRefresh is similar to the FileTree provider with the refresh feature enabled.
+// Updates can be handled with the `Watch()` function.
+func FileTreeRefresh(dirname string) {
+	DefaultStore.FileTreeRefresh(dirname)
+}
+
 // FileList registers a configstore provider which reads from the files contained in the directory given in parameter.
 // The content of the files should be JSON/YAML similar to the File provider.
 func FileList(dirname string) {
 	DefaultStore.FileList(dirname)
 }
 
+// FileListRefresh is similar to the FileList provider with the refresh feature enabled.
+// Updates can be handled with the `Watch()` function.
+func FileListRefresh(dirname string) {
+	DefaultStore.FileListRefresh(dirname)
+}
+
 // InMemory registers an InMemoryProvider with a given arbitrary name and returns it.
 // You can append any number of items to it, see Add().
 func InMemory(name string) *InMemoryProvider {

go.mod

@@ -3,7 +3,9 @@ module github.com/ovh/configstore
 go 1.12
 
 require (
+	github.com/fsnotify/fsnotify v1.5.1
 	github.com/ghodss/yaml v1.0.0
 	github.com/stretchr/testify v1.6.1
+	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -1,12 +1,17 @@
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

provider_filetree.go

@@ -1,42 +1,127 @@
 package configstore
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"unicode"
 	"unicode/utf8"
+
+	"github.com/fsnotify/fsnotify"
 )
 
 func fileTreeProvider(s *Store, dirname string) {
+	fileTree(s, dirname, false)
+}
+
+func fileTreeRefreshProvider(s *Store, dirname string) {
+	fileTree(s, dirname, true)
+}
+
+func fileTree(s *Store, dirname string, refresh bool) {
 	if dirname == "" {
 		return
 	}
 
-	providername := fmt.Sprintf("filetree:%s", dirname)
+	providername := buildProviderName("filetree", refresh, dirname)
 
-	files, err := ioutil.ReadDir(dirname)
+	items, err := loadItems(dirname)
+	if err != nil {
+		errorProvider(s, providername, err)
+		return
+	}
+
+	inmem := inMemoryProvider(s, providername)
+	inmem.mut.Lock()
+	inmem.items = items
+	inmem.mut.Unlock()
+
+	if !refresh {
+		return
+	}
+
+	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		errorProvider(s, providername, err)
 		return
 	}
 
+	go func() {
+		defer watcher.Close()
+
+		for {
+			select {
+			case <-s.ctx.Done():
+				return
+
+			case event, ok := <-watcher.Events:
+				if !ok {
+					continue
+				}
+
+				// We don't care about chmods
+				if event.Op&fsnotify.Chmod != 0 {
+					continue
+				}
+
+				// Create event will be trigger -- avoid two refreshes
+				if event.Op&fsnotify.Rename != 0 {
+					continue
+				}
+
+				// Add new path if it's a directory
+				if event.Op&fsnotify.Create != 0 {
+					if err := watchDirectory(watcher, event.Name); err != nil {
+						logError(err)
+					}
+				}
+
+				// We can't stat a deleted path, then we always
+				// remove old path even if it's not a directory
+				if event.Op&fsnotify.Remove != 0 {
+					_ = watcher.Remove(event.Name)
+				}
+
+				items, err := loadItems(dirname)
+				if err != nil {
+					logError(err)
+				} else {
+					inmem.mut.Lock()
+					inmem.items = items
+					inmem.mut.Unlock()
+					s.NotifyWatchers()
+				}
+
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					continue
+				}
+				logError(err)
+			}
+		}
+	}()
+
+	if err := watchDirectory(watcher, dirname); err != nil {
+		errorProvider(s, providername, err)
+	}
+}
+
+func loadItems(dirname string) ([]Item, error) {
+	files, err := ioutil.ReadDir(dirname)
+	if err != nil {
+		return nil, err
+	}
+
 	var items []Item
 	for _, f := range files {
 		filename := filepath.Join(dirname, f.Name())
 		subitems, err := walk(filename, f)
 		if err != nil {
-			errorProvider(s, providername, err)
-			return
+			return nil, err
 		}
 		items = append(items, subitems...)
 	}
-
-	inmem := inMemoryProvider(s, providername)
-	for _, it := range items {
-		inmem.Add(it)
-	}
+	return items, nil
 }
 
 func isDir(filename string, f os.FileInfo) bool {
@@ -58,6 +143,22 @@ func isDir(filename string, f os.FileInfo) bool {
 	return f.IsDir()
 }
 
+func watchDirectory(watcher *fsnotify.Watcher, root string) error {
+	return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if isDir(path, f) {
+			if err := watcher.Add(path); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
 func walk(filename string, f os.FileInfo) ([]Item, error) {
 	if isDir(filename, f) {
 		return browseDir([]Item{}, filename, f.Name())

providers.go

@@ -8,8 +8,8 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
-	"time"
 
+	"github.com/fsnotify/fsnotify"
 	"github.com/ghodss/yaml"
 )
 
@@ -21,10 +21,14 @@ var LogInfoFunc = log.Printf
 ** DEFAULT PROVIDERS IMPLEMENTATION
  */
 
-func errorProvider(s *Store, name string, err error) {
+func logError(err error) {
 	if LogErrorFunc != nil {
 		LogErrorFunc("error: %v", err)
 	}
+}
+
+func errorProvider(s *Store, name string, err error) {
+	logError(err)
 	s.RegisterProvider(name, newErrorProvider(err))
 }
 
@@ -56,9 +60,8 @@ func file(s *Store, filename string, refresh bool, fn func([]byte) ([]Item, erro
 		return
 	}
 
-	providername := fmt.Sprintf("file:%s", filename)
+	providername := buildProviderName("file", refresh, filename)
 
-	last := time.Now()
 	vals, err := readFile(filename, fn)
 	if err != nil {
 		errorProvider(s, providername, err)
@@ -70,40 +73,73 @@ func file(s *Store, filename string, refresh bool, fn func([]byte) ([]Item, erro
 	}
 	inmem.Add(vals...)
 
-	if refresh {
-		go func() {
-			ticker := time.NewTicker(10 * time.Second)
-			for range ticker.C {
-				finfo, err := os.Stat(filename)
-				if err != nil {
+	if !refresh {
+		return
+	}
+
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		errorProvider(s, providername, err)
+		return
+	}
+
+	go func() {
+		defer watcher.Close()
+
+		for {
+			select {
+			case <-s.ctx.Done():
+				return
+
+			case event, ok := <-watcher.Events:
+				if !ok {
 					continue
 				}
-				if finfo.ModTime().After(last) {
-					last = finfo.ModTime()
-				} else {
-					continue
+
+				if event.Op&fsnotify.Write != 0 {
+					vals, err := readFile(filename, fn)
+					if err != nil {
+						logError(err)
+					} else {
+						inmem.mut.Lock()
+						inmem.items = vals
+						inmem.mut.Unlock()
+						s.NotifyWatchers()
+					}
 				}
-				vals, err := readFile(filename, fn)
-				if err != nil {
+
+			case err, ok := <-watcher.Errors:
+				if !ok {
 					continue
 				}
-				inmem.mut.Lock()
-				inmem.items = vals
-				inmem.mut.Unlock()
-				s.NotifyWatchers()
+				logError(err)
 			}
-		}()
+		}
+	}()
+
+	if err := watcher.Add(filename); err != nil {
+		errorProvider(s, providername, err)
 	}
 }
 
 func fileListProvider(s *Store, dirname string) {
+	fileList(s, dirname, false)
+}
+
+func fileListRefreshProvider(s *Store, dirname string) {
+	fileList(s, dirname, true)
+}
+
+func fileList(s *Store, dirname string, refresh bool) {
 	if dirname == "" {
 		return
 	}
 
+	providername := buildProviderName("filelist", refresh, dirname)
+
 	files, err := ioutil.ReadDir(dirname)
 	if err != nil {
-		errorProvider(s, fmt.Sprintf("filelist:%s", dirname), err)
+		errorProvider(s, providername, err)
 		return
 	}
 
@@ -114,15 +150,19 @@ func fileListProvider(s *Store, dirname string) {
 		if file.Mode()&os.ModeSymlink != 0 {
 			linkedFile, err := os.Stat(filepath.Join(dirname, file.Name()))
 			if err != nil {
-				errorProvider(s, fmt.Sprintf("filelist:%s", dirname), err)
+				errorProvider(s, providername, err)
 				return
 			}
 			if linkedFile.IsDir() {
 				continue
 			}
 		}
 
-		fileProvider(s, filepath.Join(dirname, file.Name()))
+		if refresh {
+			fileRefreshProvider(s, filepath.Join(dirname, file.Name()))
+		} else {
+			fileProvider(s, filepath.Join(dirname, file.Name()))
+		}
 	}
 }
 
@@ -195,3 +235,10 @@ func envProvider(s *Store, prefix string) {
 		}
 	}
 }
+
+func buildProviderName(name string, refresh bool, parameter string) string {
+	if refresh {
+		return fmt.Sprintf("%s+refresh:%s", name, parameter)
+	}
+	return fmt.Sprintf("%s:%s", name, parameter)
+}