From 10e2ec552b4bcfc4a399c650ef8641d42f3dddc3 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Tue, 12 Jan 2021 15:39:39 +0900 Subject: [PATCH 1/2] Add configurations for Event watcher --- pkg/config/BUILD.bazel | 2 + pkg/config/config.go | 7 + pkg/config/event_watcher.go | 137 ++++++++++++ pkg/config/event_watcher_test.go | 217 +++++++++++++++++++ pkg/config/image_watcher_test.go | 14 -- pkg/config/piped.go | 40 ++++ pkg/config/piped_test.go | 10 + pkg/config/testdata/.pipe/event-watcher.yaml | 14 ++ pkg/config/testdata/piped/piped-config.yaml | 9 + 9 files changed, 436 insertions(+), 14 deletions(-) create mode 100644 pkg/config/event_watcher.go create mode 100644 pkg/config/event_watcher_test.go create mode 100644 pkg/config/testdata/.pipe/event-watcher.yaml diff --git a/pkg/config/BUILD.bazel b/pkg/config/BUILD.bazel index fc0a64a903..fc6c57bd61 100644 --- a/pkg/config/BUILD.bazel +++ b/pkg/config/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "deployment_lambda.go", "deployment_terraform.go", "duration.go", + "event_watcher.go", "image_watcher.go", "piped.go", "replicas.go", @@ -38,6 +39,7 @@ go_test( "deployment_kubernetes_test.go", "deployment_terraform_test.go", "deployment_test.go", + "event_watcher_test.go", "image_watcher_test.go", "piped_test.go", "replicas_test.go", diff --git a/pkg/config/config.go b/pkg/config/config.go index 0dd090312d..7a8584592e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -64,6 +64,8 @@ const ( KindAnalysisTemplate Kind = "AnalysisTemplate" // KindImageWatcher represents configuration for Repo Watcher. KindImageWatcher Kind = "ImageWatcher" + // KindEventWatcher represents configuration for Event Watcher. + KindEventWatcher Kind = "EventWatcher" ) var ( @@ -87,6 +89,7 @@ type Config struct { ControlPlaneSpec *ControlPlaneSpec AnalysisTemplateSpec *AnalysisTemplateSpec ImageWatcherSpec *ImageWatcherSpec + EventWatcherSpec *EventWatcherSpec SealedSecretSpec *SealedSecretSpec } @@ -150,6 +153,10 @@ func (c *Config) init(kind Kind, apiVersion string) error { c.ImageWatcherSpec = &ImageWatcherSpec{} c.spec = c.ImageWatcherSpec + case KindEventWatcher: + c.EventWatcherSpec = &EventWatcherSpec{} + c.spec = c.EventWatcherSpec + default: return fmt.Errorf("unsupported kind: %s", c.Kind) } diff --git a/pkg/config/event_watcher.go b/pkg/config/event_watcher.go new file mode 100644 index 0000000000..d47b3e1cae --- /dev/null +++ b/pkg/config/event_watcher.go @@ -0,0 +1,137 @@ +// Copyright 2021 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +type EventWatcherSpec struct { + Events []EventWatcherEvent `json:"events"` +} + +// EventWatcherEvent defines which file will be replaced when the given event happened. +type EventWatcherEvent struct { + // The event name. The combination of this name and labels must be unique within a project. + Name string `json:"name"` + // List of places where will be replaced when the new event found. + Replacements []EventWatcherReplacement `json:"replacements"` + // Additional attributes of event. This can make an event definition + // unique even if the one with the same name exists. + Labels map[string]string `json:"labels"` +} + +type EventWatcherReplacement struct { + // The path to the file to be updated. + File string `json:"file"` + // The yaml path to the field to be updated. It requires to start + // with `$` which represents the root element. e.g. `$.foo.bar[0].baz`. + YAMLField string `json:"yamlField"` + // TODO: Support JSONField to replace values in json format + // TODO: Support HCLField to replace values in HCL format +} + +// LoadEventWatcher gives back parsed EventWatcher config after merging config files placed under +// the .pipe directory. With "includes" and "excludes", you can filter the files included the result. +// "excludes" are prioritized if both "excludes" and "includes" are given. ErrNotFound is returned if not found. +func LoadEventWatcher(repoRoot string, includes, excludes []string) (*EventWatcherSpec, error) { + dir := filepath.Join(repoRoot, SharedConfigurationDirName) + files, err := ioutil.ReadDir(dir) + if os.IsNotExist(err) { + return nil, ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", dir, err) + } + + spec := &EventWatcherSpec{ + Events: make([]EventWatcherEvent, 0), + } + filtered, err := filterEventWatcherFiles(files, includes, excludes) + if err != nil { + return nil, fmt.Errorf("failed to filter event watcher files at %s: %w", dir, err) + } + for _, f := range filtered { + if f.IsDir() { + continue + } + path := filepath.Join(dir, f.Name()) + cfg, err := LoadFromYAML(path) + if err != nil { + return nil, fmt.Errorf("failed to load config file %s: %w", path, err) + } + if cfg.Kind == KindEventWatcher { + spec.Events = append(spec.Events, cfg.EventWatcherSpec.Events...) + } + } + if err := spec.Validate(); err != nil { + return nil, err + } + + return spec, nil +} + +// filterEventWatcherFiles filters the given files based on the given Includes and Excludes. +// Excludes are prioritized if both Excludes and Includes are given. +func filterEventWatcherFiles(files []os.FileInfo, includes, excludes []string) ([]os.FileInfo, error) { + if len(includes) == 0 && len(excludes) == 0 { + return files, nil + } + + filtered := make([]os.FileInfo, 0, len(files)) + useWhitelist := len(includes) != 0 && len(excludes) == 0 + if useWhitelist { + whiteList := make(map[string]struct{}, len(includes)) + for _, i := range includes { + whiteList[i] = struct{}{} + } + for _, f := range files { + if _, ok := whiteList[f.Name()]; ok { + filtered = append(filtered, f) + } + } + return filtered, nil + } + + blackList := make(map[string]struct{}, len(excludes)) + for _, e := range excludes { + blackList[e] = struct{}{} + } + for _, f := range files { + if _, ok := blackList[f.Name()]; !ok { + filtered = append(filtered, f) + } + } + return filtered, nil +} + +func (s *EventWatcherSpec) Validate() error { + if len(s.Events) == 0 { + return ErrNotFound + } + + for _, e := range s.Events { + if e.Name == "" { + return fmt.Errorf("event name must not be empty") + } + if len(e.Replacements) == 0 { + return fmt.Errorf("there must be at least one replacement to an event") + } + } + return nil +} diff --git a/pkg/config/event_watcher_test.go b/pkg/config/event_watcher_test.go new file mode 100644 index 0000000000..8524fb57a5 --- /dev/null +++ b/pkg/config/event_watcher_test.go @@ -0,0 +1,217 @@ +// Copyright 2021 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestLoadEventWatcher(t *testing.T) { + want := &EventWatcherSpec{Events: []EventWatcherEvent{ + { + Name: "app1-image-update", + Replacements: []EventWatcherReplacement{ + { + File: "app1/deployment.yaml", + YAMLField: "$.spec.template.spec.containers[0].image", + }, + }, + }, + { + Name: "app2-helm-release", + Replacements: []EventWatcherReplacement{ + { + File: "app2/.pipe.yaml", + YAMLField: "$.spec.input.helmChart.version", + }, + }, + Labels: map[string]string{ + "repoId": "repo-1", + }, + }, + }} + + t.Run("valid config files given", func(t *testing.T) { + got, err := LoadEventWatcher("testdata", nil, []string{"README.md"}) + assert.NoError(t, err) + assert.Equal(t, want, got) + }) +} + +func TestEventWatcherValidate(t *testing.T) { + testcases := []struct { + name string + eventWatcherSpec EventWatcherSpec + wantErr bool + }{ + { + name: "no name given", + eventWatcherSpec: EventWatcherSpec{ + Events: []EventWatcherEvent{ + { + Replacements: []EventWatcherReplacement{ + { + File: "file", + YAMLField: "$.foo", + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "no replacements given", + eventWatcherSpec: EventWatcherSpec{ + Events: []EventWatcherEvent{ + { + Name: "event-a", + }, + }, + }, + wantErr: true, + }, + { + name: "valid config given", + eventWatcherSpec: EventWatcherSpec{ + Events: []EventWatcherEvent{ + { + Name: "event-a", + Replacements: []EventWatcherReplacement{ + { + File: "file", + YAMLField: "$.foo", + }, + }, + }, + }, + }, + wantErr: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + err := tc.eventWatcherSpec.Validate() + assert.Equal(t, tc.wantErr, err != nil) + }) + } +} + +type fakeFileInfo struct { + name string +} + +func (f *fakeFileInfo) Name() string { return f.name } + +// Below methods are required to meet the interface. +func (f *fakeFileInfo) Size() int64 { return 0 } +func (f *fakeFileInfo) Mode() os.FileMode { return 0 } +func (f *fakeFileInfo) ModTime() time.Time { return time.Now() } +func (f *fakeFileInfo) IsDir() bool { return false } +func (f *fakeFileInfo) Sys() interface{} { return nil } + +func TestFilterEventWatcherFiles(t *testing.T) { + testcases := []struct { + name string + files []os.FileInfo + includes []string + excludes []string + want []os.FileInfo + wantErr bool + }{ + { + name: "both includes and excludes aren't given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + }, + want: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + }, + wantErr: false, + }, + { + name: "both includes and excludes are given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + }, + want: []os.FileInfo{}, + includes: []string{"file-1"}, + excludes: []string{"file-1"}, + wantErr: false, + }, + { + name: "includes given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + &fakeFileInfo{ + name: "file-2", + }, + &fakeFileInfo{ + name: "file-3", + }, + }, + includes: []string{"file-1", "file-3"}, + want: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + &fakeFileInfo{ + name: "file-3", + }, + }, + wantErr: false, + }, + { + name: "excludes given", + files: []os.FileInfo{ + &fakeFileInfo{ + name: "file-1", + }, + &fakeFileInfo{ + name: "file-2", + }, + &fakeFileInfo{ + name: "file-3", + }, + }, + excludes: []string{"file-1", "file-3"}, + want: []os.FileInfo{ + &fakeFileInfo{ + name: "file-2", + }, + }, + wantErr: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got, err := filterEventWatcherFiles(tc.files, tc.includes, tc.excludes) + assert.Equal(t, tc.wantErr, err != nil) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/pkg/config/image_watcher_test.go b/pkg/config/image_watcher_test.go index 44258ed241..7698e3c2b3 100644 --- a/pkg/config/image_watcher_test.go +++ b/pkg/config/image_watcher_test.go @@ -17,24 +17,10 @@ package config import ( "os" "testing" - "time" "github.com/stretchr/testify/assert" ) -type fakeFileInfo struct { - name string -} - -func (f *fakeFileInfo) Name() string { return f.name } - -// Below methods are required to meet the interface. -func (f *fakeFileInfo) Size() int64 { return 0 } -func (f *fakeFileInfo) Mode() os.FileMode { return 0 } -func (f *fakeFileInfo) ModTime() time.Time { return time.Now() } -func (f *fakeFileInfo) IsDir() bool { return false } -func (f *fakeFileInfo) Sys() interface{} { return nil } - func TestFilterImageWatcherFiles(t *testing.T) { testcases := []struct { name string diff --git a/pkg/config/piped.go b/pkg/config/piped.go index 1f39186f6d..4327d0155e 100644 --- a/pkg/config/piped.go +++ b/pkg/config/piped.go @@ -61,6 +61,8 @@ type PipedSpec struct { SealedSecretManagement *SealedSecretManagement `json:"sealedSecretManagement"` // Optional settings for image watcher. ImageWatcher PipedImageWatcher `json:"imageWatcher"` + // Optional settings for event watcher. + EventWatcher PipedEventWatcher `json:"eventWatcher"` } // Validate validates configured data of all fields. @@ -641,3 +643,41 @@ type PipedImageWatcherGitRepo struct { // This is prioritized if both includes and this are given. Excludes []string `json:"excludes"` } + +type PipedEventWatcher struct { + // Interval to fetch the latest event and compare it with one defined in replacement files. + CheckInterval Duration `json:"checkInterval"` + // Settings for each git repository. + GitRepos []PipedEventWatcherGitRepo `json:"gitRepos"` +} + +// Validate checks if: +// - empty repo ids exist +// - duplicated repository settings exist +func (p *PipedEventWatcher) Validate() error { + repos := make(map[string]struct{}, len(p.GitRepos)) + for i := 0; i < len(p.GitRepos); i++ { + if p.GitRepos[i].RepoID == "" { + return fmt.Errorf("repoId is required", p.GitRepos[i].RepoID) + } + if _, ok := repos[p.GitRepos[i].RepoID]; ok { + return fmt.Errorf("duplicated repo id (%s) found in the eventWatcher directive", p.GitRepos[i].RepoID) + } + repos[p.GitRepos[i].RepoID] = struct{}{} + } + return nil +} + +type PipedEventWatcherGitRepo struct { + // Id of the git repository. This must be unique within + // the repos' elements. + RepoID string `json:"repoId"` + // The commit message used to push after replacing values. + // Default message is used if not given. + CommitMessage string `json:"commitMessage"` + // The paths to files to be included. + Includes []string `json:"includes"` + // The paths to files to be excluded. + // This is prioritized if both includes and this one are given. + Excludes []string `json:"excludes"` +} diff --git a/pkg/config/piped_test.go b/pkg/config/piped_test.go index 328014ef80..d22088c9d5 100644 --- a/pkg/config/piped_test.go +++ b/pkg/config/piped_test.go @@ -231,6 +231,16 @@ func TestPipedConfig(t *testing.T) { }, }, }, + EventWatcher: PipedEventWatcher{ + CheckInterval: Duration(10 * time.Minute), + GitRepos: []PipedEventWatcherGitRepo{ + { + RepoID: "repo-1", + CommitMessage: "Update values by Event watcher", + Includes: []string{"event-watcher-dev.yaml", "event-watcher-stg.yaml"}, + }, + }, + }, }, expectedError: nil, }, diff --git a/pkg/config/testdata/.pipe/event-watcher.yaml b/pkg/config/testdata/.pipe/event-watcher.yaml new file mode 100644 index 0000000000..fd4b6b7c4d --- /dev/null +++ b/pkg/config/testdata/.pipe/event-watcher.yaml @@ -0,0 +1,14 @@ +apiVersion: pipecd.dev/v1beta1 +kind: EventWatcher +spec: + events: + - name: app1-image-update + replacements: + - file: app1/deployment.yaml + yamlField: $.spec.template.spec.containers[0].image + - name: app2-helm-release + replacements: + - file: app2/.pipe.yaml + yamlField: $.spec.input.helmChart.version + labels: + repoId: repo-1 diff --git a/pkg/config/testdata/piped/piped-config.yaml b/pkg/config/testdata/piped/piped-config.yaml index ca83a7f222..07cb7b319e 100644 --- a/pkg/config/testdata/piped/piped-config.yaml +++ b/pkg/config/testdata/piped/piped-config.yaml @@ -140,3 +140,12 @@ spec: includes: - imagewatcher-dev.yaml - imagewatcher-stg.yaml + + eventWatcher: + checkInterval: 10m + gitRepos: + - repoId: repo-1 + commitMessage: Update values by Event watcher + includes: + - event-watcher-dev.yaml + - event-watcher-stg.yaml From bca01b2b92f52f91dae5161934642b3306a9bff4 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Tue, 12 Jan 2021 16:58:20 +0900 Subject: [PATCH 2/2] Fix --- pkg/config/event_watcher.go | 13 ++++++------ pkg/config/event_watcher_test.go | 6 +++--- pkg/config/piped.go | 21 ++++++++++---------- pkg/config/testdata/.pipe/event-watcher.yaml | 4 ++-- 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/pkg/config/event_watcher.go b/pkg/config/event_watcher.go index d47b3e1cae..834def137b 100644 --- a/pkg/config/event_watcher.go +++ b/pkg/config/event_watcher.go @@ -27,13 +27,13 @@ type EventWatcherSpec struct { // EventWatcherEvent defines which file will be replaced when the given event happened. type EventWatcherEvent struct { - // The event name. The combination of this name and labels must be unique within a project. + // The event name. Name string `json:"name"` - // List of places where will be replaced when the new event found. - Replacements []EventWatcherReplacement `json:"replacements"` // Additional attributes of event. This can make an event definition // unique even if the one with the same name exists. Labels map[string]string `json:"labels"` + // List of places where will be replaced when the new event matches. + Replacements []EventWatcherReplacement `json:"replacements"` } type EventWatcherReplacement struct { @@ -59,6 +59,7 @@ func LoadEventWatcher(repoRoot string, includes, excludes []string) (*EventWatch return nil, fmt.Errorf("failed to read %s: %w", dir, err) } + // Start merging events defined across multiple files. spec := &EventWatcherSpec{ Events: make([]EventWatcherEvent, 0), } @@ -79,6 +80,7 @@ func LoadEventWatcher(repoRoot string, includes, excludes []string) (*EventWatch spec.Events = append(spec.Events, cfg.EventWatcherSpec.Events...) } } + if err := spec.Validate(); err != nil { return nil, err } @@ -121,10 +123,6 @@ func filterEventWatcherFiles(files []os.FileInfo, includes, excludes []string) ( } func (s *EventWatcherSpec) Validate() error { - if len(s.Events) == 0 { - return ErrNotFound - } - for _, e := range s.Events { if e.Name == "" { return fmt.Errorf("event name must not be empty") @@ -132,6 +130,7 @@ func (s *EventWatcherSpec) Validate() error { if len(e.Replacements) == 0 { return fmt.Errorf("there must be at least one replacement to an event") } + // TODO: Consider merging events if there are events whose combination of name and labels is the same } return nil } diff --git a/pkg/config/event_watcher_test.go b/pkg/config/event_watcher_test.go index 8524fb57a5..b540f0f0a0 100644 --- a/pkg/config/event_watcher_test.go +++ b/pkg/config/event_watcher_test.go @@ -35,15 +35,15 @@ func TestLoadEventWatcher(t *testing.T) { }, { Name: "app2-helm-release", + Labels: map[string]string{ + "repoId": "repo-1", + }, Replacements: []EventWatcherReplacement{ { File: "app2/.pipe.yaml", YAMLField: "$.spec.input.helmChart.version", }, }, - Labels: map[string]string{ - "repoId": "repo-1", - }, }, }} diff --git a/pkg/config/piped.go b/pkg/config/piped.go index 4327d0155e..2bea4868af 100644 --- a/pkg/config/piped.go +++ b/pkg/config/piped.go @@ -645,25 +645,24 @@ type PipedImageWatcherGitRepo struct { } type PipedEventWatcher struct { - // Interval to fetch the latest event and compare it with one defined in replacement files. + // Interval to fetch the latest event and compare it with one defined in EventWatcher config files CheckInterval Duration `json:"checkInterval"` // Settings for each git repository. GitRepos []PipedEventWatcherGitRepo `json:"gitRepos"` } -// Validate checks if: -// - empty repo ids exist -// - duplicated repository settings exist func (p *PipedEventWatcher) Validate() error { - repos := make(map[string]struct{}, len(p.GitRepos)) - for i := 0; i < len(p.GitRepos); i++ { - if p.GitRepos[i].RepoID == "" { - return fmt.Errorf("repoId is required", p.GitRepos[i].RepoID) + seen := make(map[string]struct{}, len(p.GitRepos)) + for i, repo := range p.GitRepos { + // Validate the existence of repo ID. + if repo.RepoID == "" { + return fmt.Errorf("missing repoID at index %d", i) } - if _, ok := repos[p.GitRepos[i].RepoID]; ok { - return fmt.Errorf("duplicated repo id (%s) found in the eventWatcher directive", p.GitRepos[i].RepoID) + // Validate if duplicated repository settings exist. + if _, ok := seen[repo.RepoID]; ok { + return fmt.Errorf("duplicated repo id (%s) found in the eventWatcher directive", repo.RepoID) } - repos[p.GitRepos[i].RepoID] = struct{}{} + seen[repo.RepoID] = struct{}{} } return nil } diff --git a/pkg/config/testdata/.pipe/event-watcher.yaml b/pkg/config/testdata/.pipe/event-watcher.yaml index fd4b6b7c4d..fdbe3fd5fe 100644 --- a/pkg/config/testdata/.pipe/event-watcher.yaml +++ b/pkg/config/testdata/.pipe/event-watcher.yaml @@ -7,8 +7,8 @@ spec: - file: app1/deployment.yaml yamlField: $.spec.template.spec.containers[0].image - name: app2-helm-release + labels: + repoId: repo-1 replacements: - file: app2/.pipe.yaml yamlField: $.spec.input.helmChart.version - labels: - repoId: repo-1