diff --git a/pkg/app/api/grpcapi/piped_api.go b/pkg/app/api/grpcapi/piped_api.go index fcc2a16d11..a480b25050 100644 --- a/pkg/app/api/grpcapi/piped_api.go +++ b/pkg/app/api/grpcapi/piped_api.go @@ -933,6 +933,16 @@ func (a *PipedAPI) GetDesiredVersion(ctx context.Context, _ *pipedservice.GetDes }, nil } +func (a *PipedAPI) UpdateApplicationConfigurations(ctx context.Context, req *pipedservice.UpdateApplicationConfigurationsRequest) (*pipedservice.UpdateApplicationConfigurationsResponse, error) { + // TODO: Update the given application configurations + return nil, status.Errorf(codes.Unimplemented, "UpdateApplicationConfigurations is not implemented yet") +} + +func (a *PipedAPI) ReportUnregisteredApplicationConfigurations(ctx context.Context, req *pipedservice.ReportUnregisteredApplicationConfigurationsRequest) (*pipedservice.ReportUnregisteredApplicationConfigurationsResponse, error) { + // TODO: Make the unused application configurations cache up-to-date + return nil, status.Errorf(codes.Unimplemented, "ReportUnregisteredApplicationConfigurations is not implemented yet") +} + // validateAppBelongsToPiped checks if the given application belongs to the given piped. // It gives back an error unless the application belongs to the piped. func (a *PipedAPI) validateAppBelongsToPiped(ctx context.Context, appID, pipedID string) error { diff --git a/pkg/app/api/service/pipedservice/service.proto b/pkg/app/api/service/pipedservice/service.proto index 8c19c261bd..0d42d62eee 100644 --- a/pkg/app/api/service/pipedservice/service.proto +++ b/pkg/app/api/service/pipedservice/service.proto @@ -162,7 +162,10 @@ service PipedService { // GetDesiredVersion returns the desired version of the given Piped. rpc GetDesiredVersion(GetDesiredVersionRequest) returns (GetDesiredVersionResponse) {} - // TODO: Add an rpc to update application info based on one defined in the application config + // UpdateApplicationConfigurations updates application configurations. + rpc UpdateApplicationConfigurations(UpdateApplicationConfigurationsRequest) returns (UpdateApplicationConfigurationsResponse) {} + // ReportLatestUnusedApplicationConfigurations puts the latest configurations of applications that isn't registered yet. + rpc ReportUnregisteredApplicationConfigurations(ReportUnregisteredApplicationConfigurationsRequest) returns (ReportUnregisteredApplicationConfigurationsResponse) {} } enum ListOrder { @@ -445,3 +448,21 @@ message GetDesiredVersionRequest { message GetDesiredVersionResponse { string version = 1; } + + +message UpdateApplicationConfigurationsRequest { + // The application configurations that should be updated. + repeated pipe.model.ApplicationInfo applications = 1; +} + +message UpdateApplicationConfigurationsResponse { +} + +message ReportUnregisteredApplicationConfigurationsRequest { + // All the latest application configurations that isn't registered yet. + // Note that ALL configs are always be contained every time. + repeated pipe.model.ApplicationInfo applications = 1; +} + +message ReportUnregisteredApplicationConfigurationsResponse { +} diff --git a/pkg/app/piped/apistore/environmentstore/store.go b/pkg/app/piped/apistore/environmentstore/store.go index db7fd268b7..845ad846fb 100644 --- a/pkg/app/piped/apistore/environmentstore/store.go +++ b/pkg/app/piped/apistore/environmentstore/store.go @@ -34,27 +34,33 @@ const ( type apiClient interface { GetEnvironment(ctx context.Context, in *pipedservice.GetEnvironmentRequest, opts ...grpc.CallOption) (*pipedservice.GetEnvironmentResponse, error) + //GetEnvironmentByName(ctx context.Context, in *pipedservice.GetEnvironmentByNameRequest, opts ...grpc.CallOption) (*pipedservice.GetEnvironmentByNameResponse, error) } // Lister helps list and get Environment. // All objects returned here must be treated as read-only. type Lister interface { Get(ctx context.Context, id string) (*model.Environment, error) + GetByName(ctx context.Context, name string) (*model.Environment, error) } type Store struct { apiClient apiClient - cache cache.Cache - callGroup *singleflight.Group - logger *zap.Logger + // A goroutine-safe map from id to Environment. + cache cache.Cache + // A goroutine-safe map from name to Environment. + cacheByName cache.Cache + callGroup *singleflight.Group + logger *zap.Logger } -func NewStore(apiClient apiClient, cache cache.Cache, logger *zap.Logger) *Store { +func NewStore(apiClient apiClient, cache, cacheByName cache.Cache, logger *zap.Logger) *Store { return &Store{ - apiClient: apiClient, - cache: cache, - callGroup: &singleflight.Group{}, - logger: logger.Named("environmentstore"), + apiClient: apiClient, + cache: cache, + cacheByName: cacheByName, + callGroup: &singleflight.Group{}, + logger: logger.Named("environmentstore"), } } @@ -94,3 +100,45 @@ func (s *Store) Get(ctx context.Context, id string) (*model.Environment, error) } return data.(*model.Environment), nil } + +// TODO: Implement environmentstore.GetByName +func (s *Store) GetByName(ctx context.Context, name string) (*model.Environment, error) { + /* + env, err := s.cacheByName.Get(name) + if err == nil { + return env.(*model.Environment), nil + } + + // Ensure that timeout is configured. + ctx, cancel := context.WithTimeout(ctx, defaultAPITimeout) + defer cancel() + + // Ensure that only one RPC call is executed for the given key at a time + // and the newest data is stored in the cache. + data, err, _ := s.callGroup.Do(name, func() (interface{}, error) { + req := &pipedservice.GetEnvironmentByNameRequest{ + Name: name, + } + resp, err := s.apiClient.GetEnvironmentByName(ctx, req) + if err != nil { + s.logger.Warn("failed to get environment from control plane", + zap.String("name", name), + zap.Error(err), + ) + return nil, fmt.Errorf("failed to get environment %s, %w", name, err) + } + + if err := s.cacheByName.Put(id, resp.Environment); err != nil { + s.logger.Warn("failed to put environment to cache", zap.Error(err)) + } + return resp.Environment, nil + }) + + if err != nil { + return nil, err + } + return data.(*model.Environment), nil + */ + + return nil, fmt.Errorf("not implemented") +} diff --git a/pkg/app/piped/appconfigreporter/BUILD.bazel b/pkg/app/piped/appconfigreporter/BUILD.bazel new file mode 100644 index 0000000000..971f15a722 --- /dev/null +++ b/pkg/app/piped/appconfigreporter/BUILD.bazel @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["appconfigreporter.go"], + importpath = "github.com/pipe-cd/pipe/pkg/app/piped/appconfigreporter", + visibility = ["//visibility:public"], + deps = [ + "//pkg/app/api/service/pipedservice:go_default_library", + "//pkg/config:go_default_library", + "//pkg/git:go_default_library", + "//pkg/model:go_default_library", + "@org_golang_google_grpc//:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], +) + +go_test( + name = "go_default_test", + size = "small", + srcs = ["appconfigreporter_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/model:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], +) diff --git a/pkg/app/piped/appconfigreporter/appconfigreporter.go b/pkg/app/piped/appconfigreporter/appconfigreporter.go new file mode 100644 index 0000000000..0b8cf20d8b --- /dev/null +++ b/pkg/app/piped/appconfigreporter/appconfigreporter.go @@ -0,0 +1,370 @@ +// Copyright 2021 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package appconfigreporter + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "go.uber.org/zap" + "google.golang.org/grpc" + + "github.com/pipe-cd/pipe/pkg/app/api/service/pipedservice" + "github.com/pipe-cd/pipe/pkg/config" + "github.com/pipe-cd/pipe/pkg/git" + "github.com/pipe-cd/pipe/pkg/model" +) + +type apiClient interface { + UpdateApplicationConfigurations(ctx context.Context, in *pipedservice.UpdateApplicationConfigurationsRequest, opts ...grpc.CallOption) (*pipedservice.UpdateApplicationConfigurationsResponse, error) + ReportUnregisteredApplicationConfigurations(ctx context.Context, in *pipedservice.ReportUnregisteredApplicationConfigurationsRequest, opts ...grpc.CallOption) (*pipedservice.ReportUnregisteredApplicationConfigurationsResponse, error) +} + +type gitClient interface { + Clone(ctx context.Context, repoID, remote, branch, destination string) (git.Repo, error) +} + +type gitRepo interface { + GetPath() string + ChangedFiles(ctx context.Context, from, to string) ([]string, error) +} + +type applicationLister interface { + List() []*model.Application +} + +type environmentGetter interface { + GetByName(ctx context.Context, name string) (*model.Environment, error) +} + +type fileSystem struct{} + +func (s *fileSystem) Open(name string) (fs.File, error) { return os.Open(name) } + +type Reporter struct { + apiClient apiClient + gitClient gitClient + applicationLister applicationLister + envGetter environmentGetter + config *config.PipedSpec + gitRepos map[string]git.Repo + gracePeriod time.Duration + // Cache for the last scanned commit for each repository. + lastScannedCommits map[string]string + fileSystem fs.FS + logger *zap.Logger + + // Whether it already swept all unregistered apps from control-plane. + sweptUnregisteredApps bool +} + +func NewReporter( + apiClient apiClient, + gitClient gitClient, + appLister applicationLister, + envGetter environmentGetter, + cfg *config.PipedSpec, + gracePeriod time.Duration, + logger *zap.Logger, +) *Reporter { + return &Reporter{ + apiClient: apiClient, + gitClient: gitClient, + applicationLister: appLister, + envGetter: envGetter, + config: cfg, + gracePeriod: gracePeriod, + lastScannedCommits: make(map[string]string), + fileSystem: &fileSystem{}, + logger: logger.Named("app-config-reporter"), + } +} + +func (r *Reporter) Run(ctx context.Context) error { + r.logger.Info("start running app-config-reporter") + + // Pre-clone to cache the registered git repositories. + r.gitRepos = make(map[string]git.Repo, len(r.config.Repositories)) + for _, repoCfg := range r.config.Repositories { + repo, err := r.gitClient.Clone(ctx, repoCfg.RepoID, repoCfg.Remote, repoCfg.Branch, "") + if err != nil { + r.logger.Error("failed to clone repository", + zap.String("repo-id", repoCfg.RepoID), + zap.Error(err), + ) + return err + } + r.gitRepos[repoCfg.RepoID] = repo + } + + // FIXME: Think about sync interval of app config reporter + ticker := time.NewTicker(r.config.SyncInterval.Duration()) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := r.scanAppConfigs(ctx); err != nil { + r.logger.Error("failed to check application configurations defined in Git", zap.Error(err)) + } + case <-ctx.Done(): + r.logger.Info("app-config-reporter has been stopped") + return nil + } + } +} + +// scanAppConfigs checks and reports two types of applications. +// One is applications registered in Control-plane already, and another is ones that aren't registered yet. +func (r *Reporter) scanAppConfigs(ctx context.Context) error { + if len(r.gitRepos) == 0 { + r.logger.Info("no repositories were configured for this piped") + return nil + } + + // Make all repos up-to-date. + for repoID, repo := range r.gitRepos { + if err := repo.Pull(ctx, repo.GetClonedBranch()); err != nil { + r.logger.Error("failed to update repo to latest", + zap.String("repo-id", repoID), + zap.Error(err), + ) + return err + } + } + + // Create a map to determine from GitPath if the application is registered. + apps := r.applicationLister.List() + registeredAppPaths := make(map[string]struct{}, len(apps)) + for _, app := range apps { + key := makeGitPathKey(app.GitPath.Repo.Id, filepath.Join(app.GitPath.Path, app.GitPath.ConfigFilename)) + registeredAppPaths[key] = struct{}{} + } + + if err := r.updateRegisteredApps(ctx, registeredAppPaths); err != nil { + return fmt.Errorf("failed to update registered applications: %w", err) + } + if err := r.updateUnregisteredApps(ctx, registeredAppPaths); err != nil { + return fmt.Errorf("failed to update unregistered applications: %w", err) + } + + return nil +} + +// updateRegisteredApps sends application configurations that have changed since the last time to the control-plane. +func (r *Reporter) updateRegisteredApps(ctx context.Context, registeredAppPaths map[string]struct{}) (err error) { + apps := make([]*model.ApplicationInfo, 0) + headCommits := make(map[string]string, len(r.gitRepos)) + for repoID, repo := range r.gitRepos { + var headCommit git.Commit + headCommit, err = repo.GetLatestCommit(ctx) + if err != nil { + return fmt.Errorf("failed to get the latest commit of %s: %w", repoID, err) + } + // Skip if the head commit is already scanned. + lastScannedCommit, ok := r.lastScannedCommits[repoID] + if ok && headCommit.Hash == lastScannedCommit { + continue + } + var as []*model.ApplicationInfo + as, err = r.findRegisteredApps(ctx, repoID, repo, lastScannedCommit, headCommit.Hash, registeredAppPaths) + if err != nil { + return err + } + apps = append(apps, as...) + headCommits[repoID] = headCommit.Hash + } + defer func() { + if err == nil { + for repoID, hash := range headCommits { + r.lastScannedCommits[repoID] = hash + } + } + }() + if len(apps) == 0 { + return + } + + _, err = r.apiClient.UpdateApplicationConfigurations( + ctx, + &pipedservice.UpdateApplicationConfigurationsRequest{ + Applications: apps, + }, + ) + if err != nil { + return fmt.Errorf("failed to update application configurations: %w", err) + } + return +} + +// findRegisteredApps finds out registered application info in the given git repository. +func (r *Reporter) findRegisteredApps(ctx context.Context, repoID string, repo gitRepo, lastScannedCommit, headCommitHash string, registeredAppPaths map[string]struct{}) ([]*model.ApplicationInfo, error) { + if lastScannedCommit == "" { + return r.scanAllFiles(ctx, repo.GetPath(), repoID, registeredAppPaths, true) + } + + files, err := repo.ChangedFiles(ctx, lastScannedCommit, headCommitHash) + if err != nil { + return nil, fmt.Errorf("failed to get files those were touched between two commits: %w", err) + } + if len(files) == 0 { + // The case where all changes have been fully reverted. + return []*model.ApplicationInfo{}, nil + } + apps := make([]*model.ApplicationInfo, 0) + for _, filename := range files { + if !strings.HasSuffix(filename, model.DefaultDeploymentConfigFileExtension) { + continue + } + gitPathKey := makeGitPathKey(repoID, filename) + if _, registered := registeredAppPaths[gitPathKey]; !registered { + continue + } + appInfo, err := r.readApplicationInfo(ctx, filepath.Join(repo.GetPath(), filename)) + if err != nil { + r.logger.Error("failed to read application info", + zap.String("repo-id", repoID), + zap.String("config-file-path", filename), + zap.Error(err), + ) + continue + } + apps = append(apps, appInfo) + } + return apps, nil +} + +// updateUnregisteredApps sends all unregistered application configurations to the control-plane. +func (r *Reporter) updateUnregisteredApps(ctx context.Context, registeredAppPaths map[string]struct{}) error { + apps := make([]*model.ApplicationInfo, 0) + for repoID, repo := range r.gitRepos { + as, err := r.findUnregisteredApps(ctx, repo.GetPath(), repoID, registeredAppPaths) + if err != nil { + return err + } + r.logger.Info(fmt.Sprintf("found out %d unregistered applications in repository %s", len(as), repoID)) + apps = append(apps, as...) + } + if len(apps) == 0 { + if r.sweptUnregisteredApps { + return nil + } + r.sweptUnregisteredApps = true + } else { + r.sweptUnregisteredApps = false + } + + _, err := r.apiClient.ReportUnregisteredApplicationConfigurations( + ctx, + &pipedservice.ReportUnregisteredApplicationConfigurationsRequest{ + Applications: apps, + }, + ) + if err != nil { + return fmt.Errorf("failed to put the latest unregistered application configurations: %w", err) + } + return nil +} + +// findUnregisteredApps finds out unregistered application info in the given git repository. +func (r *Reporter) findUnregisteredApps(ctx context.Context, repoPath, repoID string, registeredAppPaths map[string]struct{}) ([]*model.ApplicationInfo, error) { + return r.scanAllFiles(ctx, repoPath, repoID, registeredAppPaths, false) +} + +// scanAllFiles inspects all files under the root or the given repository. +// And gives back all application info as much as possible. +func (r *Reporter) scanAllFiles(ctx context.Context, repoRoot, repoID string, registeredAppPaths map[string]struct{}, wantRegistered bool) ([]*model.ApplicationInfo, error) { + apps := make([]*model.ApplicationInfo, 0) + err := fs.WalkDir(r.fileSystem, repoRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if !strings.HasSuffix(path, model.DefaultDeploymentConfigFileExtension) { + return nil + } + + cfgRelPath, err := filepath.Rel(repoRoot, path) + if err != nil { + return err + } + + gitPathKey := makeGitPathKey(repoID, cfgRelPath) + if _, registered := registeredAppPaths[gitPathKey]; registered != wantRegistered { + return nil + } + + appInfo, err := r.readApplicationInfo(ctx, path) + if err != nil { + r.logger.Error("failed to read application info", + zap.String("repo-id", repoID), + zap.String("config-file-path", cfgRelPath), + zap.Error(err), + ) + return nil + } + apps = append(apps, appInfo) + // Continue reading so that it can return apps as much as possible. + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to inspect files under %s: %w", repoRoot, err) + } + return apps, nil +} + +// makeGitPathKey builds a unique path between repositories. +// cfgFilePath is a relative path from the repo root. +func makeGitPathKey(repoID, cfgFilePath string) string { + return fmt.Sprintf("%s:%s", repoID, cfgFilePath) +} + +func (r *Reporter) readApplicationInfo(ctx context.Context, cfgFilePath string) (*model.ApplicationInfo, error) { + b, err := fs.ReadFile(r.fileSystem, cfgFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open the configuration file: %w", err) + } + cfg, err := config.DecodeYAML(b) + if err != nil { + return nil, fmt.Errorf("failed to decode configuration file: %w", err) + } + + spec, ok := cfg.GetGenericDeployment() + if !ok { + return nil, fmt.Errorf("unsupported application kind %q", cfg.Kind) + } + env, err := r.envGetter.GetByName(ctx, spec.EnvName) + if err != nil { + return nil, fmt.Errorf("failed to get env by name: %w", err) + } + + // TODO: Return an error if any one of required field of Application is empty + return &model.ApplicationInfo{ + Name: spec.Name, + // TODO: Convert Kind string into dedicated type + //Kind: cfg.Kind, + EnvId: env.Id, + Path: filepath.Dir(cfgFilePath), + ConfigFilename: filepath.Base(cfgFilePath), + Labels: spec.Labels, + }, nil +} diff --git a/pkg/app/piped/appconfigreporter/appconfigreporter_test.go b/pkg/app/piped/appconfigreporter/appconfigreporter_test.go new file mode 100644 index 0000000000..d3d6fd4e27 --- /dev/null +++ b/pkg/app/piped/appconfigreporter/appconfigreporter_test.go @@ -0,0 +1,305 @@ +// Copyright 2021 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package appconfigreporter + +import ( + "context" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + "github.com/pipe-cd/pipe/pkg/model" +) + +type fakeEnvGetter struct { + env *model.Environment + err error +} + +func (f fakeEnvGetter) Get(_ context.Context, _ string) (*model.Environment, error) { + return f.env, f.err +} + +func (f fakeEnvGetter) GetByName(_ context.Context, _ string) (*model.Environment, error) { + return f.env, f.err +} + +func TestReporter_findUnregisteredApps(t *testing.T) { + type args struct { + registeredAppPaths map[string]struct{} + repoPath, repoID string + } + testcases := []struct { + name string + reporter *Reporter + args args + want []*model.ApplicationInfo + wantErr bool + }{ + { + name: "file not found", + reporter: &Reporter{ + envGetter: &fakeEnvGetter{env: &model.Environment{Id: "env-1"}}, + fileSystem: fstest.MapFS{ + "path/to/repo-1/app-1/app.pipecd.yaml": &fstest.MapFile{Data: []byte("")}, + }, + logger: zap.NewNop(), + }, + args: args{ + repoPath: "invalid", + repoID: "repo-1", + registeredAppPaths: map[string]struct{}{}, + }, + want: nil, + wantErr: true, + }, + { + name: "all are registered", + reporter: &Reporter{ + envGetter: &fakeEnvGetter{env: &model.Environment{Id: "env-1"}}, + fileSystem: fstest.MapFS{ + "path/to/repo-1/app-1/app.pipecd.yaml": &fstest.MapFile{Data: []byte("")}, + }, + logger: zap.NewNop(), + }, + args: args{ + repoPath: "path/to/repo-1", + repoID: "repo-1", + registeredAppPaths: map[string]struct{}{ + "repo-1:app-1/app.pipecd.yaml": {}, + }, + }, + want: []*model.ApplicationInfo{}, + wantErr: false, + }, + { + name: "invalid app config is contained", + reporter: &Reporter{ + envGetter: &fakeEnvGetter{env: &model.Environment{Id: "env-1"}}, + fileSystem: fstest.MapFS{ + "path/to/repo-1/app-1/app.pipecd.yaml": &fstest.MapFile{Data: []byte("invalid-text")}, + }, + logger: zap.NewNop(), + }, + args: args{ + repoPath: "path/to/repo-1", + repoID: "repo-1", + registeredAppPaths: map[string]struct{}{}, + }, + want: []*model.ApplicationInfo{}, + wantErr: false, + }, + { + name: "valid app config that is unregistered", + reporter: &Reporter{ + envGetter: &fakeEnvGetter{env: &model.Environment{Id: "env-1"}}, + fileSystem: fstest.MapFS{ + "path/to/repo-1/app-1/app.pipecd.yaml": &fstest.MapFile{Data: []byte(` +apiVersion: pipecd.dev/v1beta1 +kind: KubernetesApp +spec: + name: app-1 + labels: + key-1: value-1`)}, + }, + logger: zap.NewNop(), + }, + args: args{ + repoPath: "path/to/repo-1", + repoID: "repo-1", + registeredAppPaths: map[string]struct{}{}, + }, + want: []*model.ApplicationInfo{ + { + Name: "app-1", + EnvId: "env-1", + Labels: map[string]string{"key-1": "value-1"}, + Path: "path/to/repo-1/app-1", + ConfigFilename: "app.pipecd.yaml", + }, + }, + wantErr: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got, err := tc.reporter.findUnregisteredApps(context.Background(), tc.args.repoPath, tc.args.repoID, tc.args.registeredAppPaths) + assert.Equal(t, tc.wantErr, err != nil) + assert.Equal(t, tc.want, got) + }) + } +} + +type fakeGitRepo struct { + path string + changedFiles []string + err error +} + +func (f *fakeGitRepo) GetPath() string { + return f.path +} + +func (f *fakeGitRepo) ChangedFiles(_ context.Context, _, _ string) ([]string, error) { + return f.changedFiles, f.err +} + +func TestReporter_findRegisteredApps(t *testing.T) { + type args struct { + repoID string + repo gitRepo + lastScannedCommit string + registeredAppPaths map[string]struct{} + } + testcases := []struct { + name string + reporter *Reporter + args args + want []*model.ApplicationInfo + wantErr bool + }{ + { + name: "no changed file", + reporter: &Reporter{ + envGetter: &fakeEnvGetter{env: &model.Environment{Id: "env-1"}}, + fileSystem: fstest.MapFS{ + "path/to/repo-1/app-1/app.pipecd.yaml": &fstest.MapFile{Data: []byte("invalid-text")}, + }, + logger: zap.NewNop(), + }, + args: args{ + repoID: "repo-1", + repo: &fakeGitRepo{path: "path/to/repo-1", changedFiles: nil}, + lastScannedCommit: "xxx", + }, + want: []*model.ApplicationInfo{}, + wantErr: false, + }, + { + name: "all are unregistered", + reporter: &Reporter{ + envGetter: &fakeEnvGetter{env: &model.Environment{Id: "env-1"}}, + fileSystem: fstest.MapFS{ + "path/to/repo-1/app-1/app.pipecd.yaml": &fstest.MapFile{Data: []byte("")}, + }, + logger: zap.NewNop(), + }, + args: args{ + repoID: "repo-1", + repo: &fakeGitRepo{path: "path/to/repo-1", changedFiles: []string{"app-1/app.pipecd.yaml"}}, + lastScannedCommit: "xxx", + }, + want: []*model.ApplicationInfo{}, + wantErr: false, + }, + { + name: "invalid app config is contained", + reporter: &Reporter{ + envGetter: &fakeEnvGetter{env: &model.Environment{Id: "env-1"}}, + fileSystem: fstest.MapFS{ + "path/to/repo-1/app-1/app.pipecd.yaml": &fstest.MapFile{Data: []byte("invalid-text")}, + }, + logger: zap.NewNop(), + }, + args: args{ + repoID: "repo-1", + repo: &fakeGitRepo{path: "path/to/repo-1", changedFiles: []string{"app-1/app.pipecd.yaml"}}, + lastScannedCommit: "xxx", + registeredAppPaths: map[string]struct{}{ + "repo-1:app-1/app.pipecd.yaml": {}, + }, + }, + want: []*model.ApplicationInfo{}, + wantErr: false, + }, + { + name: "valid app config that is registered", + reporter: &Reporter{ + envGetter: &fakeEnvGetter{env: &model.Environment{Id: "env-1"}}, + fileSystem: fstest.MapFS{ + "path/to/repo-1/app-1/app.pipecd.yaml": &fstest.MapFile{Data: []byte(` +apiVersion: pipecd.dev/v1beta1 +kind: KubernetesApp +spec: + name: app-1 + labels: + key-1: value-1`)}, + }, + logger: zap.NewNop(), + }, + args: args{ + repoID: "repo-1", + repo: &fakeGitRepo{path: "path/to/repo-1", changedFiles: []string{"app-1/app.pipecd.yaml"}}, + lastScannedCommit: "xxx", + registeredAppPaths: map[string]struct{}{ + "repo-1:app-1/app.pipecd.yaml": {}, + }, + }, + want: []*model.ApplicationInfo{ + { + Name: "app-1", + EnvId: "env-1", + Labels: map[string]string{"key-1": "value-1"}, + Path: "path/to/repo-1/app-1", + ConfigFilename: "app.pipecd.yaml", + }, + }, + wantErr: false, + }, + { + name: "last commit commit is empty", + reporter: &Reporter{ + envGetter: &fakeEnvGetter{env: &model.Environment{Id: "env-1"}}, + fileSystem: fstest.MapFS{ + "path/to/repo-1/app-1/app.pipecd.yaml": &fstest.MapFile{Data: []byte(` +apiVersion: pipecd.dev/v1beta1 +kind: KubernetesApp +spec: + name: app-1 + labels: + key-1: value-1`)}, + }, + logger: zap.NewNop(), + }, + args: args{ + repoID: "repo-1", + repo: &fakeGitRepo{path: "path/to/repo-1"}, + lastScannedCommit: "", + registeredAppPaths: map[string]struct{}{ + "repo-1:app-1/app.pipecd.yaml": {}, + }, + }, + want: []*model.ApplicationInfo{ + { + Name: "app-1", + EnvId: "env-1", + Labels: map[string]string{"key-1": "value-1"}, + Path: "path/to/repo-1/app-1", + ConfigFilename: "app.pipecd.yaml", + }, + }, + wantErr: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got, err := tc.reporter.findRegisteredApps(context.Background(), tc.args.repoID, tc.args.repo, tc.args.lastScannedCommit, "", tc.args.registeredAppPaths) + assert.Equal(t, tc.wantErr, err != nil) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/pkg/app/piped/cmd/piped/BUILD.bazel b/pkg/app/piped/cmd/piped/BUILD.bazel index 02c4f629b7..4c1f18988c 100644 --- a/pkg/app/piped/cmd/piped/BUILD.bazel +++ b/pkg/app/piped/cmd/piped/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//pkg/app/piped/apistore/deploymentstore:go_default_library", "//pkg/app/piped/apistore/environmentstore:go_default_library", "//pkg/app/piped/apistore/eventstore:go_default_library", + "//pkg/app/piped/appconfigreporter:go_default_library", "//pkg/app/piped/chartrepo:go_default_library", "//pkg/app/piped/cloudprovider/kubernetes/kubernetesmetrics:go_default_library", "//pkg/app/piped/controller:go_default_library", diff --git a/pkg/app/piped/cmd/piped/piped.go b/pkg/app/piped/cmd/piped/piped.go index e2d3121d6e..18c44ae2e4 100644 --- a/pkg/app/piped/cmd/piped/piped.go +++ b/pkg/app/piped/cmd/piped/piped.go @@ -45,6 +45,7 @@ import ( "github.com/pipe-cd/pipe/pkg/app/piped/apistore/deploymentstore" "github.com/pipe-cd/pipe/pkg/app/piped/apistore/environmentstore" "github.com/pipe-cd/pipe/pkg/app/piped/apistore/eventstore" + "github.com/pipe-cd/pipe/pkg/app/piped/appconfigreporter" "github.com/pipe-cd/pipe/pkg/app/piped/chartrepo" k8scloudprovidermetrics "github.com/pipe-cd/pipe/pkg/app/piped/cloudprovider/kubernetes/kubernetesmetrics" "github.com/pipe-cd/pipe/pkg/app/piped/controller" @@ -249,6 +250,7 @@ func (p *piped) run(ctx context.Context, input cli.Input) (runErr error) { environmentStore := environmentstore.NewStore( apiClient, memorycache.NewTTLCache(ctx, 10*time.Minute, time.Minute), + memorycache.NewTTLCache(ctx, 10*time.Minute, time.Minute), input.Logger, ) @@ -439,6 +441,23 @@ func (p *piped) run(ctx context.Context, input cli.Input) (runErr error) { }) } + // Start running app-config-reporter. + { + r := appconfigreporter.NewReporter( + apiClient, + gitClient, + applicationLister, + environmentStore, + cfg, + p.gracePeriod, + input.Logger, + ) + + group.Go(func() error { + return r.Run(ctx) + }) + } + // Wait until all piped components have finished. // A terminating signal or a finish of any components // could trigger the finish of piped. diff --git a/pkg/config/deployment.go b/pkg/config/deployment.go index e9659c0435..8d4fca3e14 100644 --- a/pkg/config/deployment.go +++ b/pkg/config/deployment.go @@ -29,6 +29,10 @@ const ( ) type GenericDeploymentSpec struct { + Name string `json:"name"` + EnvName string `json:"envName"` + Labels map[string]string `json:"labels"` + // Configuration used while planning deployment. Planner DeploymentPlanner `json:"planner"` // Forcibly use QuickSync or Pipeline when commit message matched the specified pattern. diff --git a/pkg/model/application.go b/pkg/model/application.go index 57ad96b827..7a77320f20 100644 --- a/pkg/model/application.go +++ b/pkg/model/application.go @@ -20,7 +20,11 @@ import ( "strings" ) -const DefaultDeploymentConfigFileName = ".pipe.yaml" +const ( + // TODO: Consider changing the default application config name + DefaultDeploymentConfigFileName = ".pipe.yaml" + DefaultDeploymentConfigFileExtension = ".pipecd.yaml" +) // GetDeploymentConfigFilePath returns the path to deployment configuration file. func (p ApplicationGitPath) GetDeploymentConfigFilePath() string { diff --git a/pkg/model/common.proto b/pkg/model/common.proto index e2d47c4a15..e6437e9ba2 100644 --- a/pkg/model/common.proto +++ b/pkg/model/common.proto @@ -55,3 +55,12 @@ enum SyncStrategy { QUICK_SYNC = 1; PIPELINE = 2; } + +message ApplicationInfo { + string name = 1 [(validate.rules).string.min_len = 1]; + ApplicationKind kind = 2 [(validate.rules).enum.defined_only = true]; + string env_id = 3 [(validate.rules).string.min_len = 1]; + string path = 4 [(validate.rules).string.pattern = "^[^/].+$"]; + string config_filename = 5; + map labels = 6; +}