diff --git a/pkg/config/BUILD.bazel b/pkg/config/BUILD.bazel index fc0a64a903..fc6c57bd61 100644 --- a/pkg/config/BUILD.bazel +++ b/pkg/config/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "deployment_lambda.go", "deployment_terraform.go", "duration.go", + "event_watcher.go", "image_watcher.go", "piped.go", "replicas.go", @@ -38,6 +39,7 @@ go_test( "deployment_kubernetes_test.go", "deployment_terraform_test.go", "deployment_test.go", + "event_watcher_test.go", "image_watcher_test.go", "piped_test.go", "replicas_test.go", diff --git a/pkg/config/config.go b/pkg/config/config.go index 0dd090312d..7a8584592e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -64,6 +64,8 @@ const ( KindAnalysisTemplate Kind = "AnalysisTemplate" // KindImageWatcher represents configuration for Repo Watcher. KindImageWatcher Kind = "ImageWatcher" + // KindEventWatcher represents configuration for Event Watcher. + KindEventWatcher Kind = "EventWatcher" ) var ( @@ -87,6 +89,7 @@ type Config struct { ControlPlaneSpec *ControlPlaneSpec AnalysisTemplateSpec *AnalysisTemplateSpec ImageWatcherSpec *ImageWatcherSpec + EventWatcherSpec *EventWatcherSpec SealedSecretSpec *SealedSecretSpec } @@ -150,6 +153,10 @@ func (c *Config) init(kind Kind, apiVersion string) error { c.ImageWatcherSpec = &ImageWatcherSpec{} c.spec = c.ImageWatcherSpec + case KindEventWatcher: + c.EventWatcherSpec = &EventWatcherSpec{} + c.spec = c.EventWatcherSpec + default: return fmt.Errorf("unsupported kind: %s", c.Kind) } diff --git a/pkg/config/event_watcher.go b/pkg/config/event_watcher.go new file mode 100644 index 0000000000..834def137b --- /dev/null +++ b/pkg/config/event_watcher.go @@ -0,0 +1,136 @@ +// Copyright 2021 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +type EventWatcherSpec struct { + Events []EventWatcherEvent `json:"events"` +} + +// EventWatcherEvent defines which file will be replaced when the given event happened. +type EventWatcherEvent struct { + // The event name. + Name string `json:"name"` + // Additional attributes of event. This can make an event definition + // unique even if the one with the same name exists. + Labels map[string]string `json:"labels"` + // List of places where will be replaced when the new event matches. + Replacements []EventWatcherReplacement `json:"replacements"` +} + +type EventWatcherReplacement struct { + // The path to the file to be updated. + File string `json:"file"` + // The yaml path to the field to be updated. It requires to start + // with `$` which represents the root element. e.g. `$.foo.bar[0].baz`. + YAMLField string `json:"yamlField"` + // TODO: Support JSONField to replace values in json format + // TODO: Support HCLField to replace values in HCL format +} + +// LoadEventWatcher gives back parsed EventWatcher config after merging config files placed under +// the .pipe directory. With "includes" and "excludes", you can filter the files included the result. +// "excludes" are prioritized if both "excludes" and "includes" are given. ErrNotFound is returned if not found. +func LoadEventWatcher(repoRoot string, includes, excludes []string) (*EventWatcherSpec, error) { + dir := filepath.Join(repoRoot, SharedConfigurationDirName) + files, err := ioutil.ReadDir(dir) + if os.IsNotExist(err) { + return nil, ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", dir, err) + } + + // Start merging events defined across multiple files. + spec := &EventWatcherSpec{ + Events: make([]EventWatcherEvent, 0), + } + filtered, err := filterEventWatcherFiles(files, includes, excludes) + if err != nil { + return nil, fmt.Errorf("failed to filter event watcher files at %s: %w", dir, err) + } + for _, f := range filtered { + if f.IsDir() { + continue + } + path := filepath.Join(dir, f.Name()) + cfg, err := LoadFromYAML(path) + if err != nil { + return nil, fmt.Errorf("failed to load config file %s: %w", path, err) + } + if cfg.Kind == KindEventWatcher { + spec.Events = append(spec.Events, cfg.EventWatcherSpec.Events...) + } + } + + if err := spec.Validate(); err != nil { + return nil, err + } + + return spec, nil +} + +// filterEventWatcherFiles filters the given files based on the given Includes and Excludes. +// Excludes are prioritized if both Excludes and Includes are given. +func filterEventWatcherFiles(files []os.FileInfo, includes, excludes []string) ([]os.FileInfo, error) { + if len(includes) == 0 && len(excludes) == 0 { + return files, nil + } + + filtered := make([]os.FileInfo, 0, len(files)) + useWhitelist := len(includes) != 0 && len(excludes) == 0 + if useWhitelist { + whiteList := make(map[string]struct{}, len(includes)) + for _, i := range includes { + whiteList[i] = struct{}{} + } + for _, f := range files { + if _, ok := whiteList[f.Name()]; ok { + filtered = append(filtered, f) + } + } + return filtered, nil + } + + blackList := make(map[string]struct{}, len(excludes)) + for _, e := range excludes { + blackList[e] = struct{}{} + } + for _, f := range files { + if _, ok := blackList[f.Name()]; !ok { + filtered = append(filtered, f) + } + } + return filtered, nil +} + +func (s *EventWatcherSpec) Validate() error { + for _, e := range s.Events { + if e.Name == "" { + return fmt.Errorf("event name must not be empty") + } + if len(e.Replacements) == 0 { + return fmt.Errorf("there must be at least one replacement to an event") + } + // TODO: Consider merging events if there are events whose combination of name and labels is the same + } + return nil +} diff --git a/pkg/config/event_watcher_test.go b/pkg/config/event_watcher_test.go new file mode 100644 index 0000000000..b540f0f0a0 --- /dev/null +++ b/pkg/config/event_watcher_test.go @@ -0,0 +1,217 @@ +// Copyright 2021 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestLoadEventWatcher(t *testing.T) { + want := &EventWatcherSpec{Events: []EventWatcherEvent{ + { + Name: "app1-image-update", + Replacements: []EventWatcherReplacement{ + { + File: "app1/deployment.yaml", + YAMLField: "$.spec.template.spec.containers[0].image", + }, + }, + }, + { + Name: "app2-helm-release", + Labels: map[string]string{ + "repoId": "repo-1", + }, + Replacements: []EventWatcherReplacement{ + { + File: "app2/.pipe.yaml", + YAMLField: "$.spec.input.helmChart.version", + }, + }, + }, + }} + + t.Run("valid config files given", func(t *testing.T) { + got, err := LoadEventWatcher("testdata", nil, []string{"README.md"}) + assert.NoError(t, err) + assert.Equal(t, want, got) + }) +} + +func TestEventWatcherValidate(t *testing.T) { + testcases := []struct { + name string + eventWatcherSpec EventWatcherSpec + wantErr bool + }{ + { + name: "no name given", + eventWatcherSpec: EventWatcherSpec{ + Events: []EventWatcherEvent{ + { + Replacements: []EventWatcherReplacement{ + { + File: "file", + YAMLField: "$.foo", + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "no replacements given", + eventWatcherSpec: EventWatcherSpec{ + Events: []EventWatcherEvent{ + { + Name: "event-a", + }, + }, + }, + wantErr: true, + }, + { + name: "valid config given", + eventWatcherSpec: EventWatcherSpec{ + Events: []EventWatcherEvent{ + { + Name: "event-a", + Replacements: []EventWatcherReplacement{ + { + File: "file", + YAMLField: "$.foo", + }, + }, + }, + }, + }, + wantErr: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + err := tc.eventWatcherSpec.Validate() + assert.Equal(t, tc.wantErr, err != nil) + }) + } +} + +type fakeFileInfo struct { + name string +} + +func (f *fakeFileInfo) Name() string { return f.name } + +// Below methods are required to meet the interface. +func (f *fakeFileInfo) Size() int64 { return 0 } +func (f *fakeFileInfo) Mode() os.FileMode { return 0 } +func (f *fakeFileInfo) ModTime() time.Time { return time.Now() } +func (f *fakeFileInfo) IsDir() bool { return false } +func (f *fakeFileInfo) Sys() interface{} { return nil } + +func TestFilterEventWatcherFiles(t *testing.T) { + testcases := []struct { + name string + files []os.FileInfo + includes []string + excludes []string + want []os.FileInfo + wantErr bool + }{ + { + name: "both includes and excludes aren't given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + }, + want: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + }, + wantErr: false, + }, + { + name: "both includes and excludes are given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + }, + want: []os.FileInfo{}, + includes: []string{"file-1"}, + excludes: []string{"file-1"}, + wantErr: false, + }, + { + name: "includes given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + &fakeFileInfo{ + name: "file-2", + }, + &fakeFileInfo{ + name: "file-3", + }, + }, + includes: []string{"file-1", "file-3"}, + want: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + &fakeFileInfo{ + name: "file-3", + }, + }, + wantErr: false, + }, + { + name: "excludes given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + &fakeFileInfo{ + name: "file-2", + }, + &fakeFileInfo{ + name: "file-3", + }, + }, + excludes: []string{"file-1", "file-3"}, + want: []os.FileInfo{ + &fakeFileInfo{ + name: "file-2", + }, + }, + wantErr: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got, err := filterEventWatcherFiles(tc.files, tc.includes, tc.excludes) + assert.Equal(t, tc.wantErr, err != nil) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/pkg/config/image_watcher_test.go b/pkg/config/image_watcher_test.go index 44258ed241..7698e3c2b3 100644 --- a/pkg/config/image_watcher_test.go +++ b/pkg/config/image_watcher_test.go @@ -17,24 +17,10 @@ package config import ( "os" "testing" - "time" "github.com/stretchr/testify/assert" ) -type fakeFileInfo struct { - name string -} - -func (f *fakeFileInfo) Name() string { return f.name } - -// Below methods are required to meet the interface. -func (f *fakeFileInfo) Size() int64 { return 0 } -func (f *fakeFileInfo) Mode() os.FileMode { return 0 } -func (f *fakeFileInfo) ModTime() time.Time { return time.Now() } -func (f *fakeFileInfo) IsDir() bool { return false } -func (f *fakeFileInfo) Sys() interface{} { return nil } - func TestFilterImageWatcherFiles(t *testing.T) { testcases := []struct { name string diff --git a/pkg/config/piped.go b/pkg/config/piped.go index 1f39186f6d..2bea4868af 100644 --- a/pkg/config/piped.go +++ b/pkg/config/piped.go @@ -61,6 +61,8 @@ type PipedSpec struct { SealedSecretManagement *SealedSecretManagement `json:"sealedSecretManagement"` // Optional settings for image watcher. ImageWatcher PipedImageWatcher `json:"imageWatcher"` + // Optional settings for event watcher. + EventWatcher PipedEventWatcher `json:"eventWatcher"` } // Validate validates configured data of all fields. @@ -641,3 +643,40 @@ type PipedImageWatcherGitRepo struct { // This is prioritized if both includes and this are given. Excludes []string `json:"excludes"` } + +type PipedEventWatcher struct { + // Interval to fetch the latest event and compare it with one defined in EventWatcher config files + CheckInterval Duration `json:"checkInterval"` + // Settings for each git repository. + GitRepos []PipedEventWatcherGitRepo `json:"gitRepos"` +} + +func (p *PipedEventWatcher) Validate() error { + seen := make(map[string]struct{}, len(p.GitRepos)) + for i, repo := range p.GitRepos { + // Validate the existence of repo ID. + if repo.RepoID == "" { + return fmt.Errorf("missing repoID at index %d", i) + } + // Validate if duplicated repository settings exist. + if _, ok := seen[repo.RepoID]; ok { + return fmt.Errorf("duplicated repo id (%s) found in the eventWatcher directive", repo.RepoID) + } + seen[repo.RepoID] = struct{}{} + } + return nil +} + +type PipedEventWatcherGitRepo struct { + // Id of the git repository. This must be unique within + // the repos' elements. + RepoID string `json:"repoId"` + // The commit message used to push after replacing values. + // Default message is used if not given. + CommitMessage string `json:"commitMessage"` + // The paths to files to be included. + Includes []string `json:"includes"` + // The paths to files to be excluded. + // This is prioritized if both includes and this one are given. + Excludes []string `json:"excludes"` +} diff --git a/pkg/config/piped_test.go b/pkg/config/piped_test.go index 328014ef80..d22088c9d5 100644 --- a/pkg/config/piped_test.go +++ b/pkg/config/piped_test.go @@ -231,6 +231,16 @@ func TestPipedConfig(t *testing.T) { }, }, }, + EventWatcher: PipedEventWatcher{ + CheckInterval: Duration(10 * time.Minute), + GitRepos: []PipedEventWatcherGitRepo{ + { + RepoID: "repo-1", + CommitMessage: "Update values by Event watcher", + Includes: []string{"event-watcher-dev.yaml", "event-watcher-stg.yaml"}, + }, + }, + }, }, expectedError: nil, }, diff --git a/pkg/config/testdata/.pipe/event-watcher.yaml b/pkg/config/testdata/.pipe/event-watcher.yaml new file mode 100644 index 0000000000..fdbe3fd5fe --- /dev/null +++ b/pkg/config/testdata/.pipe/event-watcher.yaml @@ -0,0 +1,14 @@ +apiVersion: pipecd.dev/v1beta1 +kind: EventWatcher +spec: + events: + - name: app1-image-update + replacements: + - file: app1/deployment.yaml + yamlField: $.spec.template.spec.containers[0].image + - name: app2-helm-release + labels: + repoId: repo-1 + replacements: + - file: app2/.pipe.yaml + yamlField: $.spec.input.helmChart.version diff --git a/pkg/config/testdata/piped/piped-config.yaml b/pkg/config/testdata/piped/piped-config.yaml index ca83a7f222..07cb7b319e 100644 --- a/pkg/config/testdata/piped/piped-config.yaml +++ b/pkg/config/testdata/piped/piped-config.yaml @@ -140,3 +140,12 @@ spec: includes: - imagewatcher-dev.yaml - imagewatcher-stg.yaml + + eventWatcher: + checkInterval: 10m + gitRepos: + - repoId: repo-1 + commitMessage: Update values by Event watcher + includes: + - event-watcher-dev.yaml + - event-watcher-stg.yaml