diff --git a/pkg/app/piped/imagewatcher/watcher.go b/pkg/app/piped/imagewatcher/watcher.go index 38d4345ddc..c03ff57fed 100644 --- a/pkg/app/piped/imagewatcher/watcher.go +++ b/pkg/app/piped/imagewatcher/watcher.go @@ -114,11 +114,16 @@ func (w *watcher) run(ctx context.Context, provider imageprovider.Provider, inte updates = append(updates, u...) } if len(updates) == 0 { - w.logger.Info("no image to be updated") + w.logger.Info("no image to be updated", + zap.String("image-provider", provider.Name()), + ) continue } if err := update(updates); err != nil { - w.logger.Error("failed to update image", zap.Error(err)) + w.logger.Error("failed to update image", + zap.String("image-provider", provider.Name()), + zap.Error(err), + ) continue } } @@ -136,14 +141,13 @@ func (w *watcher) determineUpdates(ctx context.Context, repoID string, repo git. } // Load Image Watcher Config for the given repo. - includes := make([]string, 0) - excludes := make([]string, 0) + var includes, excludes []string for _, target := range w.config.ImageWatcher.Repos { - if target.RepoID != repoID { - continue + if target.RepoID == repoID { + includes = target.Includes + excludes = target.Excludes + break } - includes = append(includes, target.Includes...) - excludes = append(excludes, target.Excludes...) } cfg, ok, err := config.LoadImageWatcher(repo.GetPath(), includes, excludes) if err != nil { diff --git a/pkg/config/BUILD.bazel b/pkg/config/BUILD.bazel index 49e90326f3..65cf47f73c 100644 --- a/pkg/config/BUILD.bazel +++ b/pkg/config/BUILD.bazel @@ -38,6 +38,7 @@ go_test( "deployment_kubernetes_test.go", "deployment_terraform_test.go", "deployment_test.go", + "image_watcher_test.go", "piped_test.go", "replicas_test.go", "sealed_secret_test.go", @@ -47,6 +48,7 @@ go_test( deps = [ "//pkg/model:go_default_library", "@com_github_golang_protobuf//proto:go_default_library", + "@com_github_magiconair_properties//assert:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", "@com_github_stretchr_testify//require:go_default_library", ], diff --git a/pkg/config/image_watcher.go b/pkg/config/image_watcher.go index 1feecbf002..ba7f5a4d91 100644 --- a/pkg/config/image_watcher.go +++ b/pkg/config/image_watcher.go @@ -14,6 +14,13 @@ package config +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + type ImageWatcherSpec struct { Targets []ImageWatcherTarget `json:"targets"` } @@ -25,12 +32,79 @@ type ImageWatcherTarget struct { Field string `json:"field"` } -// LoadImageWatcher finds the config files for the image watcher in the .pipe directory first up. -// And returns parsed config, False is returned as the second returned value if not found. +// LoadImageWatcher finds the config files for the image watcher in the .pipe +// directory first up. And returns parsed config after merging the targets. +// Only one of includes or excludes can be used. +// False is returned as the second returned value if not found. func LoadImageWatcher(repoRoot string, includes, excludes []string) (*ImageWatcherSpec, bool, error) { - // TODO: Load image watcher config - // referring to AnalysisTemplateSpec - return nil, false, nil + dir := filepath.Join(repoRoot, SharedConfigurationDirName) + files, err := ioutil.ReadDir(dir) + if os.IsNotExist(err) { + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf("failed to read %s: %w", dir, err) + } + + spec := &ImageWatcherSpec{ + Targets: make([]ImageWatcherTarget, 0), + } + filtered, err := filterImageWatcherFiles(files, includes, excludes) + if err != nil { + return nil, false, fmt.Errorf("failed to filter image watcher files at %s: %w", dir, err) + } + for _, f := range filtered { + if f.IsDir() { + continue + } + path := filepath.Join(dir, f.Name()) + cfg, err := LoadFromYAML(path) + if err != nil { + return nil, false, fmt.Errorf("failed to load config file %s: %w", path, err) + } + if cfg.Kind == KindImageWatcher { + spec.Targets = append(spec.Targets, cfg.ImageWatcherSpec.Targets...) + } + } + if len(spec.Targets) == 0 { + return nil, false, nil + } + + return spec, true, nil +} + +// filterImageWatcherFiles filters the given files based on the given Includes and Excludes. +// Excludes are prioritized if both Excludes and Includes are given. +func filterImageWatcherFiles(files []os.FileInfo, includes, excludes []string) ([]os.FileInfo, error) { + if len(includes) == 0 && len(excludes) == 0 { + return files, nil + } + + filtered := make([]os.FileInfo, 0, len(files)) + useWhitelist := len(includes) != 0 && len(excludes) == 0 + if useWhitelist { + whiteList := make(map[string]struct{}, len(includes)) + for _, i := range includes { + whiteList[i] = struct{}{} + } + for _, f := range files { + if _, ok := whiteList[f.Name()]; ok { + filtered = append(filtered, f) + } + } + return filtered, nil + } + + blackList := make(map[string]struct{}, len(excludes)) + for _, e := range excludes { + blackList[e] = struct{}{} + } + for _, f := range files { + if _, ok := blackList[f.Name()]; !ok { + filtered = append(filtered, f) + } + } + return filtered, nil } func (s *ImageWatcherSpec) Validate() error { diff --git a/pkg/config/image_watcher_test.go b/pkg/config/image_watcher_test.go new file mode 100644 index 0000000000..d34409e7b8 --- /dev/null +++ b/pkg/config/image_watcher_test.go @@ -0,0 +1,126 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type fakeFileInfo struct { + name string +} + +func (f *fakeFileInfo) Name() string { return f.name } + +// Below methods are required to meet the interface. +func (f *fakeFileInfo) Size() int64 { return 0 } +func (f *fakeFileInfo) Mode() os.FileMode { return 0 } +func (f *fakeFileInfo) ModTime() time.Time { return time.Now() } +func (f *fakeFileInfo) IsDir() bool { return false } +func (f *fakeFileInfo) Sys() interface{} { return nil } + +func TestFilterImageWatcherFiles(t *testing.T) { + testcases := []struct { + name string + files []os.FileInfo + includes []string + excludes []string + want []os.FileInfo + wantErr bool + }{ + { + name: "both includes and excludes aren't given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + }, + want: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + }, + wantErr: false, + }, + { + name: "both includes and excludes are given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + }, + want: []os.FileInfo{}, + includes: []string{"file-1"}, + excludes: []string{"file-1"}, + wantErr: false, + }, + { + name: "includes given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + &fakeFileInfo{ + name: "file-2", + }, + &fakeFileInfo{ + name: "file-3", + }, + }, + includes: []string{"file-1", "file-3"}, + want: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + &fakeFileInfo{ + name: "file-3", + }, + }, + wantErr: false, + }, + { + name: "excludes given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + &fakeFileInfo{ + name: "file-2", + }, + &fakeFileInfo{ + name: "file-3", + }, + }, + excludes: []string{"file-1", "file-3"}, + want: []os.FileInfo{ + &fakeFileInfo{ + name: "file-2", + }, + }, + wantErr: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got, err := filterImageWatcherFiles(tc.files, tc.includes, tc.excludes) + assert.Equal(t, tc.wantErr, err != nil) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/pkg/config/piped.go b/pkg/config/piped.go index 7adb2bdfb9..cab4b1ea25 100644 --- a/pkg/config/piped.go +++ b/pkg/config/piped.go @@ -88,6 +88,9 @@ func (s *PipedSpec) Validate() error { return err } } + if err := s.ImageWatcher.Validate(); err != nil { + return err + } return nil } @@ -590,10 +593,23 @@ type PipedImageWatcher struct { Repos []PipedImageWatcherRepoTarget `json:"repos"` } +// Validate checks if the duplicated repository setting exists. +func (i *PipedImageWatcher) Validate() error { + repos := make(map[string]struct{}) + for _, repo := range i.Repos { + if _, ok := repos[repo.RepoID]; ok { + return fmt.Errorf("duplicated repo id (%s) found in the imageWatcher directive", repo.RepoID) + } + repos[repo.RepoID] = struct{}{} + } + return nil +} + type PipedImageWatcherRepoTarget struct { RepoID string `json:"repoId"` // The paths to ImageWatcher files to be included. Includes []string `json:"includes"` // The paths to ImageWatcher files to be excluded. + // This is prioritized if both includes and this are given. Excludes []string `json:"excludes"` } diff --git a/pkg/config/piped_test.go b/pkg/config/piped_test.go index 4f6350fe92..6ef0332443 100644 --- a/pkg/config/piped_test.go +++ b/pkg/config/piped_test.go @@ -229,7 +229,7 @@ func TestPipedConfig(t *testing.T) { Repos: []PipedImageWatcherRepoTarget{ { RepoID: "foo", - Includes: []string{".pipe/imagewatcher-dev.yaml"}, + Includes: []string{"imagewatcher-dev.yaml", "imagewatcher-stg.yaml"}, }, }, }, @@ -249,3 +249,42 @@ func TestPipedConfig(t *testing.T) { }) } } + +func TestPipedImageWatcherValidate(t *testing.T) { + testcases := []struct { + name string + imageWatcher PipedImageWatcher + wantErr bool + }{ + { + name: "duplicated repo exists", + wantErr: true, + imageWatcher: PipedImageWatcher{Repos: []PipedImageWatcherRepoTarget{ + { + RepoID: "foo", + }, + { + RepoID: "foo", + }, + }}, + }, + { + name: "repos are unique", + wantErr: false, + imageWatcher: PipedImageWatcher{Repos: []PipedImageWatcherRepoTarget{ + { + RepoID: "foo", + }, + { + RepoID: "bar", + }, + }}, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + err := tc.imageWatcher.Validate() + assert.Equal(t, tc.wantErr, err != nil) + }) + } +} diff --git a/pkg/config/testdata/piped/piped-config.yaml b/pkg/config/testdata/piped/piped-config.yaml index b8986dca65..946b1d1c6b 100644 --- a/pkg/config/testdata/piped/piped-config.yaml +++ b/pkg/config/testdata/piped/piped-config.yaml @@ -140,4 +140,5 @@ spec: repos: - repoId: foo includes: - - .pipe/imagewatcher-dev.yaml \ No newline at end of file + - imagewatcher-dev.yaml + - imagewatcher-stg.yaml