Ensure to load image watcher config

pkg/app/piped/imagewatcher/watcher.go

@@ -114,11 +114,16 @@ func (w *watcher) run(ctx context.Context, provider imageprovider.Provider, inte
 				updates = append(updates, u...)
 			}
 			if len(updates) == 0 {
-				w.logger.Info("no image to be updated")
+				w.logger.Info("no image to be updated",
+					zap.String("image-provider", provider.Name()),
+				)
 				continue
 			}
 			if err := update(updates); err != nil {
-				w.logger.Error("failed to update image", zap.Error(err))
+				w.logger.Error("failed to update image",
+					zap.String("image-provider", provider.Name()),
+					zap.Error(err),
+				)
 				continue
 			}
 		}
@@ -136,14 +141,13 @@ func (w *watcher) determineUpdates(ctx context.Context, repoID string, repo git.
 	}
 
 	// Load Image Watcher Config for the given repo.
-	includes := make([]string, 0)
-	excludes := make([]string, 0)
+	var includes, excludes []string
 	for _, target := range w.config.ImageWatcher.Repos {
-		if target.RepoID != repoID {
-			continue
+		if target.RepoID == repoID {
+			includes = target.Includes
+			excludes = target.Excludes
+			break
 		}
-		includes = append(includes, target.Includes...)
-		excludes = append(excludes, target.Excludes...)
 	}
 	cfg, ok, err := config.LoadImageWatcher(repo.GetPath(), includes, excludes)
 	if err != nil {

pkg/config/BUILD.bazel

@@ -38,6 +38,7 @@ go_test(
         "deployment_kubernetes_test.go",
         "deployment_terraform_test.go",
         "deployment_test.go",
+        "image_watcher_test.go",
         "piped_test.go",
         "replicas_test.go",
         "sealed_secret_test.go",
@@ -47,6 +48,7 @@ go_test(
     deps = [
         "//pkg/model:go_default_library",
         "@com_github_golang_protobuf//proto:go_default_library",
+        "@com_github_magiconair_properties//assert:go_default_library",
         "@com_github_stretchr_testify//assert:go_default_library",
         "@com_github_stretchr_testify//require:go_default_library",
     ],

pkg/config/image_watcher.go

@@ -14,6 +14,13 @@
 
 package config
 
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
 type ImageWatcherSpec struct {
 	Targets []ImageWatcherTarget `json:"targets"`
 }
@@ -25,12 +32,79 @@ type ImageWatcherTarget struct {
 	Field    string `json:"field"`
 }
 
-// LoadImageWatcher finds the config files for the image watcher in the .pipe directory first up.
-// And returns parsed config, False is returned as the second returned value if not found.
+// LoadImageWatcher finds the config files for the image watcher in the .pipe
+// directory first up. And returns parsed config after merging the targets.
+// Only one of includes or excludes can be used.
+// False is returned as the second returned value if not found.
 func LoadImageWatcher(repoRoot string, includes, excludes []string) (*ImageWatcherSpec, bool, error) {
-	// TODO: Load image watcher config
-	//   referring to AnalysisTemplateSpec
-	return nil, false, nil
+	dir := filepath.Join(repoRoot, SharedConfigurationDirName)
+	files, err := ioutil.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return nil, false, nil
+	}
+	if err != nil {
+		return nil, false, fmt.Errorf("failed to read %s: %w", dir, err)
+	}
+
+	spec := &ImageWatcherSpec{
+		Targets: make([]ImageWatcherTarget, 0),
+	}
+	filtered, err := filterImageWatcherFiles(files, includes, excludes)
+	if err != nil {
+		return nil, false, fmt.Errorf("failed to filter image watcher files at %s: %w", dir, err)
+	}
+	for _, f := range filtered {
+		if f.IsDir() {
+			continue
+		}
+		path := filepath.Join(dir, f.Name())
+		cfg, err := LoadFromYAML(path)
+		if err != nil {
+			return nil, false, fmt.Errorf("failed to load config file %s: %w", path, err)
+		}
+		if cfg.Kind == KindImageWatcher {
+			spec.Targets = append(spec.Targets, cfg.ImageWatcherSpec.Targets...)
+		}
+	}
+	if len(spec.Targets) == 0 {
+		return nil, false, nil
+	}
+
+	return spec, true, nil
+}
+
+// filterImageWatcherFiles filters the given files based on the given Includes and Excludes.
+// Excludes are prioritized if both Excludes and Includes are given.
+func filterImageWatcherFiles(files []os.FileInfo, includes, excludes []string) ([]os.FileInfo, error) {
+	if len(includes) == 0 && len(excludes) == 0 {
+		return files, nil
+	}
+
+	filtered := make([]os.FileInfo, 0, len(files))
+	useWhitelist := len(includes) != 0 && len(excludes) == 0
+	if useWhitelist {
+		whiteList := make(map[string]struct{}, len(includes))
+		for _, i := range includes {
+			whiteList[i] = struct{}{}
+		}
+		for _, f := range files {
+			if _, ok := whiteList[f.Name()]; ok {
+				filtered = append(filtered, f)
+			}
+		}
+		return filtered, nil
+	}
+
+	blackList := make(map[string]struct{}, len(excludes))
+	for _, e := range excludes {
+		blackList[e] = struct{}{}
+	}
+	for _, f := range files {
+		if _, ok := blackList[f.Name()]; !ok {
+			filtered = append(filtered, f)
+		}
+	}
+	return filtered, nil
 }
 
 func (s *ImageWatcherSpec) Validate() error {

pkg/config/image_watcher_test.go

@@ -0,0 +1,126 @@
+// Copyright 2020 The PipeCD Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type fakeFileInfo struct {
+	name string
+}
+
+func (f *fakeFileInfo) Name() string { return f.name }
+
+// Below methods are required to meet the interface.
+func (f *fakeFileInfo) Size() int64        { return 0 }
+func (f *fakeFileInfo) Mode() os.FileMode  { return 0 }
+func (f *fakeFileInfo) ModTime() time.Time { return time.Now() }
+func (f *fakeFileInfo) IsDir() bool        { return false }
+func (f *fakeFileInfo) Sys() interface{}   { return nil }
+
+func TestFilterImageWatcherFiles(t *testing.T) {
+	testcases := []struct {
+		name     string
+		files    []os.FileInfo
+		includes []string
+		excludes []string
+		want     []os.FileInfo
+		wantErr  bool
+	}{
+		{
+			name: "both includes and excludes aren't given",
+			files: []os.FileInfo{
+				&fakeFileInfo{
+					name: "file-1",
+				},
+			},
+			want: []os.FileInfo{
+				&fakeFileInfo{
+					name: "file-1",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "both includes and excludes are given",
+			files: []os.FileInfo{
+				&fakeFileInfo{
+					name: "file-1",
+				},
+			},
+			want:     []os.FileInfo{},
+			includes: []string{"file-1"},
+			excludes: []string{"file-1"},
+			wantErr:  false,
+		},
+		{
+			name: "includes given",
+			files: []os.FileInfo{
+				&fakeFileInfo{
+					name: "file-1",
+				},
+				&fakeFileInfo{
+					name: "file-2",
+				},
+				&fakeFileInfo{
+					name: "file-3",
+				},
+			},
+			includes: []string{"file-1", "file-3"},
+			want: []os.FileInfo{
+				&fakeFileInfo{
+					name: "file-1",
+				},
+				&fakeFileInfo{
+					name: "file-3",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "excludes given",
+			files: []os.FileInfo{
+				&fakeFileInfo{
+					name: "file-1",
+				},
+				&fakeFileInfo{
+					name: "file-2",
+				},
+				&fakeFileInfo{
+					name: "file-3",
+				},
+			},
+			excludes: []string{"file-1", "file-3"},
+			want: []os.FileInfo{
+				&fakeFileInfo{
+					name: "file-2",
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := filterImageWatcherFiles(tc.files, tc.includes, tc.excludes)
+			assert.Equal(t, tc.wantErr, err != nil)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}

pkg/config/piped.go

@@ -88,6 +88,9 @@ func (s *PipedSpec) Validate() error {
 			return err
 		}
 	}
+	if err := s.ImageWatcher.Validate(); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -590,10 +593,23 @@ type PipedImageWatcher struct {
 	Repos []PipedImageWatcherRepoTarget `json:"repos"`
 }
 
+// Validate checks if the duplicated repository setting exists.
+func (i *PipedImageWatcher) Validate() error {
+	repos := make(map[string]struct{})
+	for _, repo := range i.Repos {
+		if _, ok := repos[repo.RepoID]; ok {
+			return fmt.Errorf("duplicated repo id (%s) found in the imageWatcher directive", repo.RepoID)
+		}
+		repos[repo.RepoID] = struct{}{}
+	}
+	return nil
+}
+
 type PipedImageWatcherRepoTarget struct {
 	RepoID string `json:"repoId"`
 	// The paths to ImageWatcher files to be included.
 	Includes []string `json:"includes"`
 	// The paths to ImageWatcher files to be excluded.
+	// This is prioritized if both includes and this are given.
 	Excludes []string `json:"excludes"`
 }

pkg/config/piped_test.go

@@ -229,7 +229,7 @@ func TestPipedConfig(t *testing.T) {
 					Repos: []PipedImageWatcherRepoTarget{
 						{
 							RepoID:   "foo",
-							Includes: []string{".pipe/imagewatcher-dev.yaml"},
+							Includes: []string{"imagewatcher-dev.yaml", "imagewatcher-stg.yaml"},
 						},
 					},
 				},
@@ -249,3 +249,42 @@ func TestPipedConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestPipedImageWatcherValidate(t *testing.T) {
+	testcases := []struct {
+		name         string
+		imageWatcher PipedImageWatcher
+		wantErr      bool
+	}{
+		{
+			name:    "duplicated repo exists",
+			wantErr: true,
+			imageWatcher: PipedImageWatcher{Repos: []PipedImageWatcherRepoTarget{
+				{
+					RepoID: "foo",
+				},
+				{
+					RepoID: "foo",
+				},
+			}},
+		},
+		{
+			name:    "repos are unique",
+			wantErr: false,
+			imageWatcher: PipedImageWatcher{Repos: []PipedImageWatcherRepoTarget{
+				{
+					RepoID: "foo",
+				},
+				{
+					RepoID: "bar",
+				},
+			}},
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.imageWatcher.Validate()
+			assert.Equal(t, tc.wantErr, err != nil)
+		})
+	}
+}

pkg/config/testdata/piped/piped-config.yaml

@@ -140,4 +140,5 @@ spec:
     repos:
       - repoId: foo
         includes:
-          - .pipe/imagewatcher-dev.yaml
+          - imagewatcher-dev.yaml
+          - imagewatcher-stg.yaml