diff --git a/pkg/app/pipectl/cmd/planpreview/planpreview.go b/pkg/app/pipectl/cmd/planpreview/planpreview.go index ade52140ba..ae3fd12987 100644 --- a/pkg/app/pipectl/cmd/planpreview/planpreview.go +++ b/pkg/app/pipectl/cmd/planpreview/planpreview.go @@ -199,13 +199,15 @@ func convert(results []*model.PlanPreviewCommandResult) ReadableResult { out.FailureApplications = append(out.FailureApplications, FailureApplication{ ApplicationInfo: appInfo, Reason: a.Error, + PlanDetails: string(a.PlanDetails), }) continue } out.Applications = append(out.Applications, ApplicationResult{ ApplicationInfo: appInfo, SyncStrategy: a.SyncStrategy.String(), - Changes: string(a.Changes), + PlanSummary: string(a.PlanSummary), + PlanDetails: string(a.PlanDetails), }) } } @@ -222,7 +224,8 @@ type ReadableResult struct { type ApplicationResult struct { ApplicationInfo SyncStrategy string // QUICK_SYNC, PIPELINE - Changes string + PlanSummary string + PlanDetails string } type FailurePiped struct { @@ -232,7 +235,8 @@ type FailurePiped struct { type FailureApplication struct { ApplicationInfo - Reason string + Reason string + PlanDetails string } type PipedInfo struct { @@ -267,7 +271,8 @@ func (r ReadableResult) String() string { for i, app := range r.Applications { fmt.Fprintf(&b, "\n%d. app: %s, env: %s, kind: %s\n", i+1, app.ApplicationName, app.EnvName, app.ApplicationKind) fmt.Fprintf(&b, " sync strategy: %s\n", app.SyncStrategy) - fmt.Fprintf(&b, " changes: %s\n", app.Changes) + fmt.Fprintf(&b, " summary: %s\n", app.PlanSummary) + fmt.Fprintf(&b, " details:\n\n ---DETAILS_BEGIN---\n%s\n ---DETAILS_END---\n", app.PlanDetails) } } @@ -280,6 +285,9 @@ func (r ReadableResult) String() string { for i, app := range r.FailureApplications { fmt.Fprintf(&b, "\n%d. app: %s, env: %s, kind: %s\n", i+1, app.ApplicationName, app.EnvName, app.ApplicationKind) fmt.Fprintf(&b, " reason: %s\n", app.Reason) + if len(app.PlanDetails) > 0 { + fmt.Fprintf(&b, " details:\n\n ---DETAILS_BEGIN---\n%s\n ---DETAILS_END---\n", app.PlanDetails) + } } } diff --git a/pkg/app/pipectl/cmd/planpreview/planpreview_test.go b/pkg/app/pipectl/cmd/planpreview/planpreview_test.go index 052055da5d..61f3ab6990 100644 --- a/pkg/app/pipectl/cmd/planpreview/planpreview_test.go +++ b/pkg/app/pipectl/cmd/planpreview/planpreview_test.go @@ -51,7 +51,8 @@ There are no applications to build plan-preview ApplicationKind: model.ApplicationKind_KUBERNETES, EnvName: "env-1", SyncStrategy: model.SyncStrategy_QUICK_SYNC, - Changes: []byte("changes-1"), + PlanSummary: []byte("2 manifests will be added, 1 manifest will be deleted and 5 manifests will be changed"), + PlanDetails: []byte("changes-1"), }, }, }, @@ -61,7 +62,12 @@ Here are plan-preview for 1 application: 1. app: app-1, env: env-1, kind: KUBERNETES sync strategy: QUICK_SYNC - changes: changes-1 + summary: 2 manifests will be added, 1 manifest will be deleted and 5 manifests will be changed + details: + + ---DETAILS_BEGIN--- +changes-1 + ---DETAILS_END--- `, }, { @@ -128,7 +134,8 @@ NOTE: An error occurred while building plan-preview for applications of the foll ApplicationKind: model.ApplicationKind_KUBERNETES, EnvName: "env-1", SyncStrategy: model.SyncStrategy_QUICK_SYNC, - Changes: []byte("changes-1"), + PlanSummary: []byte("2 manifests will be added, 1 manifest will be deleted and 5 manifests will be changed"), + PlanDetails: []byte("changes-1"), }, { ApplicationId: "app-2", @@ -137,7 +144,8 @@ NOTE: An error occurred while building plan-preview for applications of the foll ApplicationKind: model.ApplicationKind_TERRAFORM, EnvName: "env-2", SyncStrategy: model.SyncStrategy_PIPELINE, - Changes: []byte("changes-2"), + PlanSummary: []byte("1 to add, 2 to change, 0 to destroy"), + PlanDetails: []byte("changes-2"), }, { ApplicationId: "app-3", @@ -169,11 +177,21 @@ Here are plan-preview for 2 applications: 1. app: app-1, env: env-1, kind: KUBERNETES sync strategy: QUICK_SYNC - changes: changes-1 + summary: 2 manifests will be added, 1 manifest will be deleted and 5 manifests will be changed + details: + + ---DETAILS_BEGIN--- +changes-1 + ---DETAILS_END--- 2. app: app-2, env: env-2, kind: TERRAFORM sync strategy: PIPELINE - changes: changes-2 + summary: 1 to add, 2 to change, 0 to destroy + details: + + ---DETAILS_BEGIN--- +changes-2 + ---DETAILS_END--- NOTE: An error occurred while building plan-preview for the following 2 applications: diff --git a/pkg/app/piped/cmd/piped/piped.go b/pkg/app/piped/cmd/piped/piped.go index fbcf0f4e7d..95f6c01201 100644 --- a/pkg/app/piped/cmd/piped/piped.go +++ b/pkg/app/piped/cmd/piped/piped.go @@ -382,10 +382,13 @@ func (p *piped) run(ctx context.Context, t cli.Telemetry) (runErr error) { if p.enablePlanPreview { h := planpreview.NewHandler( gitClient, + apiClient, commandLister, applicationLister, environmentStore, lastTriggeredCommitGetter, + decrypter, + appManifestsCache, cfg, planpreview.WithLogger(t.Logger), ) diff --git a/pkg/app/piped/controller/planner.go b/pkg/app/piped/controller/planner.go index c28a5aa71d..c7f30c2c62 100644 --- a/pkg/app/piped/controller/planner.go +++ b/pkg/app/piped/controller/planner.go @@ -162,7 +162,10 @@ func (p *planner) Run(ctx context.Context) error { } in := pln.Input{ - Deployment: p.deployment, + ApplicationID: p.deployment.ApplicationId, + ApplicationName: p.deployment.ApplicationName, + GitPath: *p.deployment.GitPath, + Trigger: *p.deployment.Trigger, MostRecentSuccessfulCommitHash: p.lastSuccessfulCommitHash, PipedConfig: p.pipedConfig, AppManifestsCache: p.appManifestsCache, diff --git a/pkg/app/piped/planner/cloudrun/cloudrun.go b/pkg/app/piped/planner/cloudrun/cloudrun.go index e67fee1499..07aa77d50a 100644 --- a/pkg/app/piped/planner/cloudrun/cloudrun.go +++ b/pkg/app/piped/planner/cloudrun/cloudrun.go @@ -64,7 +64,7 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu // If the deployment was triggered by forcing via web UI, // we rely on the user's decision. - switch in.Deployment.Trigger.SyncStrategy { + switch in.Trigger.SyncStrategy { case model.SyncStrategy_QUICK_SYNC: out.SyncStrategy = model.SyncStrategy_QUICK_SYNC out.Stages = buildQuickSyncPipeline(cfg.Input.AutoRollback, time.Now()) diff --git a/pkg/app/piped/planner/ecs/ecs.go b/pkg/app/piped/planner/ecs/ecs.go index fb3285e4f7..8017248cff 100644 --- a/pkg/app/piped/planner/ecs/ecs.go +++ b/pkg/app/piped/planner/ecs/ecs.go @@ -64,7 +64,7 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu // If the deployment was triggered by forcing via web UI, // we rely on the user's decision. - switch in.Deployment.Trigger.SyncStrategy { + switch in.Trigger.SyncStrategy { case model.SyncStrategy_QUICK_SYNC: out.SyncStrategy = model.SyncStrategy_QUICK_SYNC out.Stages = buildQuickSyncPipeline(cfg.Input.AutoRollback, time.Now()) diff --git a/pkg/app/piped/planner/kubernetes/kubernetes.go b/pkg/app/piped/planner/kubernetes/kubernetes.go index 90953e6296..ec1ceb1f69 100644 --- a/pkg/app/piped/planner/kubernetes/kubernetes.go +++ b/pkg/app/piped/planner/kubernetes/kubernetes.go @@ -72,21 +72,21 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu } manifestCache := provider.AppManifestsCache{ - AppID: in.Deployment.ApplicationId, + AppID: in.ApplicationID, Cache: in.AppManifestsCache, Logger: in.Logger, } // Load previous deployed manifests and new manifests to compare. - newManifests, ok := manifestCache.Get(in.Deployment.Trigger.Commit.Hash) + newManifests, ok := manifestCache.Get(in.Trigger.Commit.Hash) if !ok { // When the manifests were not in the cache we have to load them. - loader := provider.NewManifestLoader(in.Deployment.ApplicationName, ds.AppDir, ds.RepoDir, in.Deployment.GitPath.ConfigFilename, cfg.Input, in.Logger) + loader := provider.NewManifestLoader(in.ApplicationName, ds.AppDir, ds.RepoDir, in.GitPath.ConfigFilename, cfg.Input, in.Logger) newManifests, err = loader.LoadManifests(ctx) if err != nil { return } - manifestCache.Put(in.Deployment.Trigger.Commit.Hash, newManifests) + manifestCache.Put(in.Trigger.Commit.Hash, newManifests) } // Determine application version from the manifests. @@ -99,7 +99,7 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu // If the deployment was triggered by forcing via web UI, // we rely on the user's decision. - switch in.Deployment.Trigger.SyncStrategy { + switch in.Trigger.SyncStrategy { case model.SyncStrategy_QUICK_SYNC: out.SyncStrategy = model.SyncStrategy_QUICK_SYNC out.Stages = buildQuickSyncPipeline(cfg.Input.AutoRollback, time.Now()) @@ -127,13 +127,13 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu // This deployment is triggered by a commit with the intent to perform pipeline. // Commit Matcher will be ignored when triggered by a command. - if p := cfg.CommitMatcher.Pipeline; p != "" && in.Deployment.Trigger.Commander == "" { + if p := cfg.CommitMatcher.Pipeline; p != "" && in.Trigger.Commander == "" { pipelineRegex, err := in.RegexPool.Get(p) if err != nil { err = fmt.Errorf("failed to compile commitMatcher.pipeline(%s): %w", p, err) return out, err } - if pipelineRegex.MatchString(in.Deployment.Trigger.Commit.Message) { + if pipelineRegex.MatchString(in.Trigger.Commit.Message) { out.SyncStrategy = model.SyncStrategy_PIPELINE out.Stages = buildProgressivePipeline(cfg.Pipeline, cfg.Input.AutoRollback, time.Now()) out.Summary = fmt.Sprintf("Sync progressively because the commit message was matching %q", p) @@ -143,13 +143,13 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu // This deployment is triggered by a commit with the intent to synchronize. // Commit Matcher will be ignored when triggered by a command. - if s := cfg.CommitMatcher.QuickSync; s != "" && in.Deployment.Trigger.Commander == "" { + if s := cfg.CommitMatcher.QuickSync; s != "" && in.Trigger.Commander == "" { syncRegex, err := in.RegexPool.Get(s) if err != nil { err = fmt.Errorf("failed to compile commitMatcher.sync(%s): %w", s, err) return out, err } - if syncRegex.MatchString(in.Deployment.Trigger.Commit.Message) { + if syncRegex.MatchString(in.Trigger.Commit.Message) { out.SyncStrategy = model.SyncStrategy_QUICK_SYNC out.Stages = buildQuickSyncPipeline(cfg.Input.AutoRollback, time.Now()) out.Summary = fmt.Sprintf("Quick sync by applying all manifests because the commit message was matching %q", s) @@ -178,7 +178,7 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu return } - loader := provider.NewManifestLoader(in.Deployment.ApplicationName, runningDs.AppDir, runningDs.RepoDir, in.Deployment.GitPath.ConfigFilename, cfg.Input, in.Logger) + loader := provider.NewManifestLoader(in.ApplicationName, runningDs.AppDir, runningDs.RepoDir, in.GitPath.ConfigFilename, cfg.Input, in.Logger) oldManifests, err = loader.LoadManifests(ctx) if err != nil { err = fmt.Errorf("failed to load previously deployed manifests: %w", err) diff --git a/pkg/app/piped/planner/lambda/lambda.go b/pkg/app/piped/planner/lambda/lambda.go index 0d8f382c62..2f21a4f6d3 100644 --- a/pkg/app/piped/planner/lambda/lambda.go +++ b/pkg/app/piped/planner/lambda/lambda.go @@ -64,7 +64,7 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu // If the deployment was triggered by forcing via web UI, // we rely on the user's decision. - switch in.Deployment.Trigger.SyncStrategy { + switch in.Trigger.SyncStrategy { case model.SyncStrategy_QUICK_SYNC: out.SyncStrategy = model.SyncStrategy_QUICK_SYNC out.Stages = buildQuickSyncPipeline(cfg.Input.AutoRollback, time.Now()) diff --git a/pkg/app/piped/planner/planner.go b/pkg/app/piped/planner/planner.go index 296404e390..0de91b0b2b 100644 --- a/pkg/app/piped/planner/planner.go +++ b/pkg/app/piped/planner/planner.go @@ -37,8 +37,10 @@ type Planner interface { } type Input struct { - // Readonly deployment model. - Deployment *model.Deployment + ApplicationID string + ApplicationName string + GitPath model.ApplicationGitPath + Trigger model.DeploymentTrigger MostRecentSuccessfulCommitHash string PipedConfig *config.PipedSpec TargetDSP deploysource.Provider diff --git a/pkg/app/piped/planner/terraform/terraform.go b/pkg/app/piped/planner/terraform/terraform.go index ccfbbda3b6..9359f46cd0 100644 --- a/pkg/app/piped/planner/terraform/terraform.go +++ b/pkg/app/piped/planner/terraform/terraform.go @@ -53,8 +53,9 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu // If the deployment was triggered by forcing via web UI, // we rely on the user's decision. - switch in.Deployment.Trigger.SyncStrategy { + switch in.Trigger.SyncStrategy { case model.SyncStrategy_QUICK_SYNC: + out.SyncStrategy = model.SyncStrategy_QUICK_SYNC out.Stages = buildQuickSyncPipeline(cfg.Input.AutoRollback, time.Now()) out.Summary = "Quick sync by automatically applying any detected changes because no pipeline was configured (forced via web)" return @@ -63,6 +64,7 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu err = fmt.Errorf("unable to force sync with pipeline because no pipeline was specified") return } + out.SyncStrategy = model.SyncStrategy_PIPELINE out.Stages = buildProgressivePipeline(cfg.Pipeline, cfg.Input.AutoRollback, time.Now()) out.Summary = "Sync with the specified progressive pipeline (forced via web)" return @@ -72,11 +74,13 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu out.Version = "N/A" if cfg.Pipeline == nil || len(cfg.Pipeline.Stages) == 0 { + out.SyncStrategy = model.SyncStrategy_QUICK_SYNC out.Stages = buildQuickSyncPipeline(cfg.Input.AutoRollback, now) out.Summary = "Quick sync by automatically applying any detected changes because no pipeline was configured" return } + out.SyncStrategy = model.SyncStrategy_PIPELINE out.Stages = buildProgressivePipeline(cfg.Pipeline, cfg.Input.AutoRollback, now) out.Summary = "Sync with the specified progressive pipeline" return diff --git a/pkg/app/piped/planpreview/BUILD.bazel b/pkg/app/piped/planpreview/BUILD.bazel index 65c1938b5c..031b12d486 100644 --- a/pkg/app/piped/planpreview/BUILD.bazel +++ b/pkg/app/piped/planpreview/BUILD.bazel @@ -5,14 +5,29 @@ go_library( srcs = [ "builder.go", "handler.go", + "kubernetesdiff.go", + "terraformdiff.go", ], importpath = "github.com/pipe-cd/pipe/pkg/app/piped/planpreview", visibility = ["//visibility:public"], deps = [ + "//pkg/app/api/service/pipedservice:go_default_library", + "//pkg/app/piped/cloudprovider/kubernetes:go_default_library", + "//pkg/app/piped/cloudprovider/terraform:go_default_library", + "//pkg/app/piped/deploysource:go_default_library", + "//pkg/app/piped/planner:go_default_library", + "//pkg/app/piped/planner/registry:go_default_library", + "//pkg/app/piped/toolregistry:go_default_library", "//pkg/app/piped/trigger:go_default_library", + "//pkg/cache:go_default_library", "//pkg/config:go_default_library", + "//pkg/diff:go_default_library", "//pkg/git:go_default_library", "//pkg/model:go_default_library", + "//pkg/regexpool:go_default_library", + "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//codes:go_default_library", + "@org_golang_google_grpc//status:go_default_library", "@org_uber_go_zap//:go_default_library", ], ) diff --git a/pkg/app/piped/planpreview/builder.go b/pkg/app/piped/planpreview/builder.go index eb47e4b8c0..a447dc3983 100644 --- a/pkg/app/piped/planpreview/builder.go +++ b/pkg/app/piped/planpreview/builder.go @@ -15,15 +15,33 @@ package planpreview import ( + "bytes" "context" "fmt" + "io/ioutil" + "os" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/pipe-cd/pipe/pkg/app/api/service/pipedservice" + "github.com/pipe-cd/pipe/pkg/app/piped/deploysource" + "github.com/pipe-cd/pipe/pkg/app/piped/planner" + "github.com/pipe-cd/pipe/pkg/app/piped/planner/registry" "github.com/pipe-cd/pipe/pkg/app/piped/trigger" + "github.com/pipe-cd/pipe/pkg/cache" "github.com/pipe-cd/pipe/pkg/config" - "github.com/pipe-cd/pipe/pkg/git" "github.com/pipe-cd/pipe/pkg/model" + "github.com/pipe-cd/pipe/pkg/regexpool" +) + +const ( + workspacePattern = "plan-preview-builder-*" +) + +var ( + defaultPlannerRegistry = registry.DefaultRegistry() ) type lastTriggeredCommitGetter interface { @@ -36,62 +54,169 @@ type Builder interface { type builder struct { gitClient gitClient + apiClient apiClient applicationLister applicationLister environmentGetter environmentGetter commitGetter lastTriggeredCommitGetter - config *config.PipedSpec + secretDecrypter secretDecrypter + appManifestsCache cache.Cache + regexPool *regexpool.Pool + pipedCfg *config.PipedSpec logger *zap.Logger + + workingDir string + repoCfg config.PipedRepository } -func newBuilder(gc gitClient, al applicationLister, eg environmentGetter, cg lastTriggeredCommitGetter, cfg *config.PipedSpec, logger *zap.Logger) *builder { +func newBuilder( + gc gitClient, + ac apiClient, + al applicationLister, + eg environmentGetter, + cg lastTriggeredCommitGetter, + sd secretDecrypter, + amc cache.Cache, + rp *regexpool.Pool, + cfg *config.PipedSpec, + logger *zap.Logger, +) *builder { + return &builder{ gitClient: gc, + apiClient: ac, applicationLister: al, environmentGetter: eg, commitGetter: cg, - config: cfg, - logger: logger.Named("planpreview-builder"), + secretDecrypter: sd, + appManifestsCache: amc, + regexPool: rp, + pipedCfg: cfg, + logger: logger.Named("plan-preview-builder"), } } func (b *builder) Build(ctx context.Context, id string, cmd model.Command_BuildPlanPreview) ([]*model.ApplicationPlanPreviewResult, error) { b.logger.Info(fmt.Sprintf("start building planpreview result for command %s", id)) + // Ensure the existence of the working directory. + workingDir, err := ioutil.TempDir("", workspacePattern) + if err != nil { + return nil, fmt.Errorf("failed to create working directory (%w)", err) + } + defer os.RemoveAll(workingDir) + b.workingDir = workingDir + // Find the registered repository in Piped config and validate the command's payload against it. - repoCfg, ok := b.config.GetRepository(cmd.RepositoryId) + repoCfg, ok := b.pipedCfg.GetRepository(cmd.RepositoryId) if !ok { return nil, fmt.Errorf("repository %s was not found in Piped config", cmd.RepositoryId) } if repoCfg.Branch != cmd.BaseBranch { - return nil, fmt.Errorf("base branch repository %s was not matched, requested %s, expected %s", cmd.RepositoryId, cmd.BaseBranch, repoCfg.Branch) + return nil, fmt.Errorf("base branch of repository %s was not matched, requested %s, expected %s", cmd.RepositoryId, cmd.BaseBranch, repoCfg.Branch) } + b.repoCfg = repoCfg // List all applications that belong to this Piped // and are placed in the given repository. apps := b.listApplications(repoCfg) if len(apps) == 0 { + b.logger.Info(fmt.Sprintf("there is no target application for command %s", id)) return nil, nil } + // Find all applications that should be triggered. + triggerApps, failedResults, err := b.findTriggerApps(ctx, apps, cmd) + if err != nil { + return nil, err + } + results := failedResults + + // Plan the trigger applications for more detailed feedback. + for _, app := range triggerApps { + // We only need the environment name + // so the returned error can be ignorable. + var envName string + if env, err := b.environmentGetter.Get(ctx, app.EnvId); err == nil { + envName = env.Name + } + + r := model.MakeApplicationPlanPreviewResult(*app, envName) + results = append(results, r) + + var preCommit string + // Find the commit of the last successful deployment. + if deploy, err := b.getMostRecentlySuccessfulDeployment(ctx, app.Id); err == nil { + preCommit = deploy.Trigger.Commit.Hash + } else if status.Code(err) != codes.NotFound { + r.Error = fmt.Sprintf("failed while finding the last successful deployment (%w)", err) + continue + } + + b.logger.Info("will decide sync strategy for a application", + zap.String("id", app.Id), + zap.String("name", app.Name), + zap.String("kind", app.Kind.String()), + ) + + strategy, err := b.plan(ctx, app, cmd, preCommit) + if err != nil { + r.Error = fmt.Sprintf("failed while planning, %v", err) + continue + } + r.SyncStrategy = strategy + + b.logger.Info("successfully decided sync strategy for a application", + zap.String("id", app.Id), + zap.String("name", app.Name), + zap.String("strategy", strategy.String()), + zap.String("kind", app.Kind.String()), + ) + + var buf bytes.Buffer + var summary string + + switch app.Kind { + case model.ApplicationKind_KUBERNETES: + summary, err = b.kubernetesDiff(ctx, app, cmd, preCommit, &buf) + case model.ApplicationKind_TERRAFORM: + summary, err = b.terraformDiff(ctx, app, cmd, &buf) + default: + // TODO: Calculating planpreview's diff for other application kinds. + err = fmt.Errorf("%s application is not implemented yet (coming soon)", app.Kind.String()) + } + + r.PlanSummary = []byte(summary) + r.PlanDetails = buf.Bytes() + if err != nil { + r.Error = fmt.Sprintf("failed while calculating diff, %v", err) + continue + } + } + + return results, nil +} + +func (b *builder) findTriggerApps(ctx context.Context, apps []*model.Application, cmd model.Command_BuildPlanPreview) (triggerApps []*model.Application, failedResults []*model.ApplicationPlanPreviewResult, err error) { // Clone the source code and checkout to the given branch, commit. - repo, err := b.gitClient.Clone(ctx, repoCfg.RepoID, repoCfg.Remote, repoCfg.Branch, "") + dir, err := ioutil.TempDir(b.workingDir, "") + if err != nil { + err = fmt.Errorf("failed to create temporary directory %w", err) + return + } + repo, err := b.gitClient.Clone(ctx, b.repoCfg.RepoID, b.repoCfg.Remote, cmd.HeadBranch, dir) if err != nil { - return nil, fmt.Errorf("failed to clone git repository %s", cmd.RepositoryId) + err = fmt.Errorf("failed to clone git repository %s", cmd.RepositoryId) + return } defer repo.Clean() - if err := repo.Checkout(ctx, cmd.HeadCommit); err != nil { - return nil, fmt.Errorf("failed to checkout the head commit %s: %w", cmd.HeadCommit, err) + err = repo.Checkout(ctx, cmd.HeadCommit) + if err != nil { + err = fmt.Errorf("failed to checkout the head commit %s: %w", cmd.HeadCommit, err) + return } - // Compared to the total number of applications, - // the number of applications that should be triggered will be very smaller - // therefore we do not explicitly specify the capacity for these slices. - triggerApps := make([]*model.Application, 0) - results := make([]*model.ApplicationPlanPreviewResult, 0) - d := trigger.NewDeterminer(repo, cmd.HeadCommit, b.commitGetter, b.logger) - for _, app := range apps { shouldTrigger, err := d.ShouldTrigger(ctx, app) if err != nil { @@ -103,8 +228,8 @@ func (b *builder) Build(ctx context.Context, id string, cmd model.Command_BuildP } r := model.MakeApplicationPlanPreviewResult(*app, envName) - r.Error = fmt.Sprintf("Failed while determining the application should be triggered or not, %v", err) - results = append(results, r) + r.Error = fmt.Sprintf("failed while determining the application should be triggered or not, %v", err) + failedResults = append(failedResults, r) continue } @@ -112,40 +237,66 @@ func (b *builder) Build(ctx context.Context, id string, cmd model.Command_BuildP triggerApps = append(triggerApps, app) } } + return +} - // All triggered applications will be passed to plan. - for _, app := range triggerApps { - // We only need the environment name - // so the returned error can be ignorable. - var envName string - if env, err := b.environmentGetter.Get(ctx, app.EnvId); err == nil { - envName = env.Name - } +func (b *builder) plan(ctx context.Context, app *model.Application, cmd model.Command_BuildPlanPreview, lastSuccessfulCommit string) (strategy model.SyncStrategy, err error) { + p, ok := defaultPlannerRegistry.Planner(app.Kind) + if !ok { + err = fmt.Errorf("application kind %s is not supported yet", app.Kind.String()) + return + } - r := model.MakeApplicationPlanPreviewResult(*app, envName) - results = append(results, r) + in := planner.Input{ + ApplicationID: app.Id, + ApplicationName: app.Name, + GitPath: *app.GitPath, + Trigger: model.DeploymentTrigger{ + Commit: &model.Commit{ + Branch: cmd.HeadBranch, + Hash: cmd.HeadCommit, + }, + Commander: "pipectl", + }, + MostRecentSuccessfulCommitHash: lastSuccessfulCommit, + PipedConfig: b.pipedCfg, + AppManifestsCache: b.appManifestsCache, + RegexPool: b.regexPool, + Logger: b.logger, + } - strategy, changes, err := b.plan(repo, app, cmd) - if err != nil { - r.Error = fmt.Sprintf("Failed while planning, %v", err) - continue - } + repoCfg := b.repoCfg + repoCfg.Branch = cmd.HeadBranch - r.SyncStrategy = strategy - r.Changes = changes - } + in.TargetDSP = deploysource.NewProvider( + b.workingDir, + repoCfg, + "target", + cmd.HeadCommit, + b.gitClient, + app.GitPath, + b.secretDecrypter, + ) - return results, nil -} + if lastSuccessfulCommit != "" { + in.RunningDSP = deploysource.NewProvider( + b.workingDir, + repoCfg, + "running", + lastSuccessfulCommit, + b.gitClient, + app.GitPath, + b.secretDecrypter, + ) + } -func (b *builder) plan(repo git.Repo, app *model.Application, cmd model.Command_BuildPlanPreview) (model.SyncStrategy, []byte, error) { - // TODO: Implement planpreview plan. - // 1. Start a planner to check what/why strategy will be used - // 2. Check what resources should be added, deleted and modified - // - Terraform app: used terraform plan command - // - Kubernetes app: calculate the diff of resources at head commit and mostRecentlySuccessfulCommit + out, err := p.Plan(ctx, in) + if err != nil { + return + } - return model.SyncStrategy_QUICK_SYNC, []byte("NOT IMPLEMENTED"), nil + strategy = out.SyncStrategy + return } func (b *builder) listApplications(repo config.PipedRepository) []*model.Application { @@ -167,3 +318,25 @@ func (b *builder) listApplications(repo config.PipedRepository) []*model.Applica return out } + +func (b *builder) getMostRecentlySuccessfulDeployment(ctx context.Context, applicationID string) (*model.ApplicationDeploymentReference, error) { + var ( + err error + resp *pipedservice.GetApplicationMostRecentDeploymentResponse + retry = pipedservice.NewRetry(3) + req = &pipedservice.GetApplicationMostRecentDeploymentRequest{ + ApplicationId: applicationID, + Status: model.DeploymentStatus_DEPLOYMENT_SUCCESS, + } + ) + + for retry.WaitNext(ctx) { + if resp, err = b.apiClient.GetApplicationMostRecentDeployment(ctx, req); err == nil { + return resp.Deployment, nil + } + if !pipedservice.Retriable(err) { + return nil, err + } + } + return nil, err +} diff --git a/pkg/app/piped/planpreview/handler.go b/pkg/app/piped/planpreview/handler.go index 03933ccb0e..7459725572 100644 --- a/pkg/app/piped/planpreview/handler.go +++ b/pkg/app/piped/planpreview/handler.go @@ -21,10 +21,14 @@ import ( "time" "go.uber.org/zap" + "google.golang.org/grpc" + "github.com/pipe-cd/pipe/pkg/app/api/service/pipedservice" + "github.com/pipe-cd/pipe/pkg/cache" "github.com/pipe-cd/pipe/pkg/config" "github.com/pipe-cd/pipe/pkg/git" "github.com/pipe-cd/pipe/pkg/model" + "github.com/pipe-cd/pipe/pkg/regexpool" ) const ( @@ -78,6 +82,10 @@ type gitClient interface { Clone(ctx context.Context, repoID, remote, branch, destination string) (git.Repo, error) } +type apiClient interface { + GetApplicationMostRecentDeployment(ctx context.Context, req *pipedservice.GetApplicationMostRecentDeploymentRequest, opts ...grpc.CallOption) (*pipedservice.GetApplicationMostRecentDeploymentResponse, error) +} + type applicationLister interface { List() []*model.Application } @@ -90,6 +98,10 @@ type commandLister interface { ListBuildPlanPreviewCommands() []model.ReportableCommand } +type secretDecrypter interface { + Decrypt(string) (string, error) +} + type Handler struct { gitClient gitClient commandLister commandLister @@ -102,7 +114,19 @@ type Handler struct { logger *zap.Logger } -func NewHandler(gc gitClient, cl commandLister, al applicationLister, eg environmentGetter, cg lastTriggeredCommitGetter, cfg *config.PipedSpec, opts ...Option) *Handler { +func NewHandler( + gc gitClient, + ac apiClient, + cl commandLister, + al applicationLister, + eg environmentGetter, + cg lastTriggeredCommitGetter, + sd secretDecrypter, + appManifestsCache cache.Cache, + cfg *config.PipedSpec, + opts ...Option, +) *Handler { + opt := &options{ workerNum: defaultWorkerNum, commandQueueBufferSize: defaultCommandQueueBufferSize, @@ -113,16 +137,19 @@ func NewHandler(gc gitClient, cl commandLister, al applicationLister, eg environ for _, o := range opts { o(opt) } + h := &Handler{ gitClient: gc, commandLister: cl, commandCh: make(chan model.ReportableCommand, opt.commandQueueBufferSize), prevCommands: map[string]struct{}{}, options: opt, - logger: opt.logger.Named("planpreview-handler"), + logger: opt.logger.Named("plan-preview-handler"), } + + regexPool := regexpool.DefaultPool() h.builderFactory = func() Builder { - return newBuilder(gc, al, eg, cg, cfg, h.logger) + return newBuilder(gc, ac, al, eg, cg, sd, appManifestsCache, regexPool, cfg, h.logger) } return h diff --git a/pkg/app/piped/planpreview/handler_test.go b/pkg/app/piped/planpreview/handler_test.go index 99d818aa00..141eeaa185 100644 --- a/pkg/app/piped/planpreview/handler_test.go +++ b/pkg/app/piped/planpreview/handler_test.go @@ -61,7 +61,7 @@ func TestHandler(t *testing.T) { var mu sync.Mutex var wg sync.WaitGroup - handler := NewHandler(nil, cl, nil, nil, nil, nil, + handler := NewHandler(nil, nil, cl, nil, nil, nil, nil, nil, nil, WithWorkerNum(2), // Use a long interval because we will directly call enqueueNewCommands function in this test. WithCommandCheckInterval(time.Hour), diff --git a/pkg/app/piped/planpreview/kubernetesdiff.go b/pkg/app/piped/planpreview/kubernetesdiff.go new file mode 100644 index 0000000000..bd997b2ff5 --- /dev/null +++ b/pkg/app/piped/planpreview/kubernetesdiff.go @@ -0,0 +1,140 @@ +// Copyright 2021 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package planpreview + +import ( + "bytes" + "context" + "fmt" + "io" + + "go.uber.org/zap" + + provider "github.com/pipe-cd/pipe/pkg/app/piped/cloudprovider/kubernetes" + "github.com/pipe-cd/pipe/pkg/app/piped/deploysource" + "github.com/pipe-cd/pipe/pkg/cache" + "github.com/pipe-cd/pipe/pkg/config" + "github.com/pipe-cd/pipe/pkg/diff" + "github.com/pipe-cd/pipe/pkg/model" +) + +func (b *builder) kubernetesDiff( + ctx context.Context, + app *model.Application, + cmd model.Command_BuildPlanPreview, + lastSuccessfulCommit string, + buf *bytes.Buffer, +) (string, error) { + + var oldManifests, newManifests []provider.Manifest + var err error + + repoCfg := config.PipedRepository{ + RepoID: b.repoCfg.RepoID, + Remote: b.repoCfg.Remote, + Branch: cmd.HeadBranch, + } + + targetDSP := deploysource.NewProvider( + b.workingDir, + repoCfg, + "target", + cmd.HeadCommit, + b.gitClient, + app.GitPath, + b.secretDecrypter, + ) + newManifests, err = loadKubernetesManifests(ctx, *app, cmd.HeadCommit, targetDSP, b.appManifestsCache, b.logger) + if err != nil { + fmt.Fprintf(buf, "failed to load kubernetes manifests at the head commit (%v)\n", err) + return "", err + } + + if lastSuccessfulCommit != "" { + runningDSP := deploysource.NewProvider( + b.workingDir, + repoCfg, + "running", + lastSuccessfulCommit, + b.gitClient, + app.GitPath, + b.secretDecrypter, + ) + oldManifests, err = loadKubernetesManifests(ctx, *app, lastSuccessfulCommit, runningDSP, b.appManifestsCache, b.logger) + if err != nil { + fmt.Fprintf(buf, "failed to load kubernetes manifests at the running commit (%v)\n", err) + return "", err + } + } + + result, err := provider.DiffList(oldManifests, newManifests, + diff.WithEquateEmpty(), + diff.WithIgnoreAddingMapKeys(), + diff.WithCompareNumberAndNumericString(), + ) + if err != nil { + fmt.Fprintf(buf, "failed to compare manifests (%v)\n", err) + return "", err + } + + if result.NoChange() { + fmt.Fprintln(buf, "No changes were detected") + return "No changes were detected", nil + } + + summary := fmt.Sprintf("%d added manifests, %d changed manifests, %d deleted manifests", len(result.Adds), len(result.Changes), len(result.Deletes)) + fmt.Fprintf(buf, "---Head Commit\n+++Last Deploy\n\n%s\n", result.DiffString()) + + return summary, nil +} + +func loadKubernetesManifests(ctx context.Context, app model.Application, commit string, dsp deploysource.Provider, manifestsCache cache.Cache, logger *zap.Logger) (manifests []provider.Manifest, err error) { + cache := provider.AppManifestsCache{ + AppID: app.Id, + Cache: manifestsCache, + Logger: logger, + } + manifests, ok := cache.Get(commit) + if ok { + return manifests, nil + } + + // When the manifests were not in the cache we have to load them. + ds, err := dsp.Get(ctx, io.Discard) + if err != nil { + return nil, err + } + + deployCfg := ds.DeploymentConfig.KubernetesDeploymentSpec + if deployCfg == nil { + return nil, fmt.Errorf("malformed deployment configuration file") + } + + loader := provider.NewManifestLoader( + app.Name, + ds.AppDir, + ds.RepoDir, + app.GitPath.ConfigFilename, + deployCfg.Input, + logger, + ) + manifests, err = loader.LoadManifests(ctx) + if err != nil { + return nil, err + } + + cache.Put(commit, manifests) + return manifests, nil +} diff --git a/pkg/app/piped/planpreview/terraformdiff.go b/pkg/app/piped/planpreview/terraformdiff.go new file mode 100644 index 0000000000..570d3de57c --- /dev/null +++ b/pkg/app/piped/planpreview/terraformdiff.go @@ -0,0 +1,121 @@ +// Copyright 2021 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package planpreview + +import ( + "bytes" + "context" + "fmt" + "io" + + terraformprovider "github.com/pipe-cd/pipe/pkg/app/piped/cloudprovider/terraform" + "github.com/pipe-cd/pipe/pkg/app/piped/deploysource" + "github.com/pipe-cd/pipe/pkg/app/piped/toolregistry" + "github.com/pipe-cd/pipe/pkg/config" + "github.com/pipe-cd/pipe/pkg/model" +) + +func (b *builder) terraformDiff( + ctx context.Context, + app *model.Application, + cmd model.Command_BuildPlanPreview, + buf *bytes.Buffer, +) (string, error) { + + cp, ok := b.pipedCfg.FindCloudProvider(app.CloudProvider, model.CloudProviderTerraform) + if !ok { + err := fmt.Errorf("cloud provider %s was not found in Piped config", app.CloudProvider) + fmt.Fprintln(buf, err.Error()) + return "", err + } + cpCfg := cp.TerraformConfig + + repoCfg := config.PipedRepository{ + RepoID: b.repoCfg.RepoID, + Remote: b.repoCfg.Remote, + Branch: cmd.HeadBranch, + } + + targetDSP := deploysource.NewProvider( + b.workingDir, + repoCfg, + "target", + cmd.HeadCommit, + b.gitClient, + app.GitPath, + b.secretDecrypter, + ) + + ds, err := targetDSP.Get(ctx, io.Discard) + if err != nil { + fmt.Fprintf(buf, "failed to prepare deploy source data at the head commit (%v)\n", err) + return "", err + } + + deployCfg := ds.DeploymentConfig.TerraformDeploymentSpec + if deployCfg == nil { + err := fmt.Errorf("missing Terraform spec field in deployment configuration") + fmt.Fprintln(buf, err.Error()) + return "", err + } + + version := deployCfg.Input.TerraformVersion + terraformPath, installed, err := toolregistry.DefaultRegistry().Terraform(ctx, version) + if err != nil { + fmt.Fprintf(buf, "unable to find the specified terraform version %q (%v)\n", version, err) + return "", err + } + if installed { + b.logger.Info(fmt.Sprintf("terraform %q has just been installed to %q because of no pre-installed binary for that version", version, terraformPath)) + } + + vars := make([]string, 0, len(cpCfg.Vars)+len(deployCfg.Input.Vars)) + vars = append(vars, cpCfg.Vars...) + vars = append(vars, deployCfg.Input.Vars...) + + executor := terraformprovider.NewTerraform(terraformPath, ds.AppDir, vars, deployCfg.Input.VarFiles) + + if err := executor.Init(ctx, buf); err != nil { + fmt.Fprintf(buf, "failed while executing terraform init (%v)\n", err) + return "", err + } + + if ws := deployCfg.Input.Workspace; ws != "" { + if err := executor.SelectWorkspace(ctx, ws); err != nil { + fmt.Fprintf(buf, "failed to select workspace %q (%v). You might need to create the workspace before using by command %q\n", + ws, + err, + "terraform workspace new "+ws, + ) + return "", err + } + fmt.Fprintf(buf, "selected workspace %q\n", ws) + } + + result, err := executor.Plan(ctx, buf) + if err != nil { + fmt.Fprintf(buf, "failed while executing terraform plan (%v)\n", err) + return "", err + } + + if result.NoChanges() { + fmt.Fprintln(buf, "No changes were detected") + return "No changes were detected", nil + } + + summary := fmt.Sprintf("%d to add, %d to change, %d to destroy", result.Adds, result.Changes, result.Destroys) + fmt.Fprintln(buf, summary) + return summary, nil +} diff --git a/pkg/model/planpreview.proto b/pkg/model/planpreview.proto index b025bc3641..9155753c7b 100644 --- a/pkg/model/planpreview.proto +++ b/pkg/model/planpreview.proto @@ -59,7 +59,8 @@ message ApplicationPlanPreviewResult { // Planpreview result. SyncStrategy sync_strategy = 30; - bytes changes = 31; + bytes plan_summary = 31; + bytes plan_details = 32; // Error while building planpreview result. string error = 40;