fix: configurable CMP tar exclusions (#9675) #9789
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -103,6 +103,7 @@ type RepoServerInitConstants struct { | |
| PauseGenerationOnFailureForRequests int | ||
| SubmoduleEnabled bool | ||
| MaxCombinedDirectoryManifestsSize resource.Quantity | ||
| CMPTarExcludedGlobs []string | ||
| } | ||
|
|
||
| // NewService returns a new instance of the Manifest service | ||
|
|
@@ -213,7 +214,7 @@ func (s *Service) ListApps(ctx context.Context, q *apiclient.ListAppsRequest) (* | |
| } | ||
|
|
||
| defer io.Close(closer) | ||
| apps, err := discovery.Discover(ctx, gitClient.Root(), q.EnabledSourceTypes, s.initConstants.CMPTarExcludedGlobs) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -465,7 +466,7 @@ func (s *Service) runManifestGenAsync(ctx context.Context, repoRoot, commitSHA, | |
| var manifestGenResult *apiclient.ManifestResponse | ||
| opContext, err := opContextSrc() | ||
| if err == nil { | ||
| manifestGenResult, err = GenerateManifests(ctx, opContext.appPath, repoRoot, commitSHA, q, false, s.gitCredsStore, s.initConstants.MaxCombinedDirectoryManifestsSize, WithCMPTarDoneChannel(ch.tarDoneCh), WithCMPTarExcludedGlobs(s.initConstants.CMPTarExcludedGlobs)) | ||
| } | ||
| if err != nil { | ||
| // If manifest generation error caching is enabled | ||
|
|
@@ -872,7 +873,8 @@ func getRepoCredential(repoCredentials []*v1alpha1.RepoCreds, repoURL string) *v | |
|
|
||
| type GenerateManifestOpt func(*generateManifestOpt) | ||
| type generateManifestOpt struct { | ||
| cmpTarDoneCh chan<- bool | ||
| cmpTarExcludedGlobs []string | ||
|
Member
There was a problem hiding this comment. @leoluz is this type designed to carry the generate params so we can keep that function signature small? If so, 1) @notfromstatefarm went the right direction here and 2) I should probably refactor to put some other things on this struct. If that's incorrect, we can refactor to pass this param directly to the function. |
||
| } | ||
|
|
||
| func newGenerateManifestOpt(opts ...GenerateManifestOpt) *generateManifestOpt { | ||
|
|
@@ -892,6 +894,14 @@ func WithCMPTarDoneChannel(ch chan<- bool) GenerateManifestOpt { | |
| } | ||
| } | ||
|
|
||
| // WithCMPTarExcludedGlobs defines globs for files to filter out when streaming the tarball | ||
| // to a CMP sidecar. | ||
| func WithCMPTarExcludedGlobs(excludedGlobs []string) GenerateManifestOpt { | ||
| return func(o *generateManifestOpt) { | ||
| o.cmpTarExcludedGlobs = excludedGlobs | ||
| } | ||
| } | ||
|
|
||
| // GenerateManifests generates manifests from a path. Overrides are applied as a side effect on the given ApplicationSource. | ||
| func GenerateManifests(ctx context.Context, appPath, repoRoot, revision string, q *apiclient.ManifestRequest, isLocal bool, gitCredsStore git.CredsStore, maxCombinedManifestQuantity resource.Quantity, opts ...GenerateManifestOpt) (*apiclient.ManifestResponse, error) { | ||
| opt := newGenerateManifestOpt(opts...) | ||
|
|
@@ -900,7 +910,7 @@ func GenerateManifests(ctx context.Context, appPath, repoRoot, revision string, | |
|
|
||
| resourceTracking := argo.NewResourceTracking() | ||
|
|
||
| appSourceType, err := GetAppSourceType(ctx, q.ApplicationSource, appPath, q.AppName, q.EnabledSourceTypes, opt.cmpTarExcludedGlobs) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -924,7 +934,7 @@ func GenerateManifests(ctx context.Context, appPath, repoRoot, revision string, | |
| if q.ApplicationSource.Plugin != nil && q.ApplicationSource.Plugin.Name != "" { | ||
| targetObjs, err = runConfigManagementPlugin(appPath, repoRoot, env, q, q.Repo.GetGitCreds(gitCredsStore)) | ||
| } else { | ||
| targetObjs, err = runConfigManagementPluginSidecars(ctx, appPath, repoRoot, env, q, q.Repo.GetGitCreds(gitCredsStore), opt.cmpTarDoneCh, opt.cmpTarExcludedGlobs) | ||
| if err != nil { | ||
| err = fmt.Errorf("plugin sidecar failed. %s", err.Error()) | ||
| } | ||
|
|
@@ -1058,7 +1068,7 @@ func mergeSourceParameters(source *v1alpha1.ApplicationSource, path, appName str | |
| } | ||
|
|
||
| // GetAppSourceType returns explicit application source type or examines a directory and determines its application source type | ||
| func GetAppSourceType(ctx context.Context, source *v1alpha1.ApplicationSource, path, appName string, enableGenerateManifests map[string]bool, tarExcludedGlobs []string) (v1alpha1.ApplicationSourceType, error) { | ||
| err := mergeSourceParameters(source, path, appName) | ||
| if err != nil { | ||
| return "", fmt.Errorf("error while parsing source parameters: %v", err) | ||
|
|
@@ -1075,7 +1085,7 @@ func GetAppSourceType(ctx context.Context, source *v1alpha1.ApplicationSource, p | |
| } | ||
| return *appSourceType, nil | ||
| } | ||
| appType, err := discovery.AppType(ctx, path, enableGenerateManifests, tarExcludedGlobs) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
|
|
@@ -1479,22 +1489,22 @@ func getPluginEnvs(envVars *v1alpha1.Env, q *apiclient.ManifestRequest, creds gi | |
| return env, nil | ||
| } | ||
|
|
||
| func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath string, envVars *v1alpha1.Env, q *apiclient.ManifestRequest, creds git.Creds, tarDoneCh chan<- bool, tarExcludedGlobs []string) ([]*unstructured.Unstructured, error) { | ||
| // compute variables. | ||
| env, err := getPluginEnvs(envVars, q, creds, true) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // detect config management plugin server (sidecar) | ||
| conn, cmpClient, err := discovery.DetectConfigManagementPlugin(ctx, appPath, env, tarExcludedGlobs) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer io.Close(conn) | ||
|
|
||
| // generate manifests using commands provided in plugin config file in detected cmp-server sidecar | ||
| cmpManifests, err := generateManifestsCMP(ctx, appPath, repoPath, env, cmpClient, tarDoneCh, tarExcludedGlobs) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error generating manifests in cmp: %s", err) | ||
| } | ||
|
|
@@ -1512,15 +1522,15 @@ func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath st | |
| // generateManifestsCMP will send the appPath files to the cmp-server over a gRPC stream. | ||
| // The cmp-server will generate the manifests. Returns a response object with the generated | ||
| // manifests. | ||
| func generateManifestsCMP(ctx context.Context, appPath, repoPath string, env []string, cmpClient pluginclient.ConfigManagementPluginServiceClient, tarDoneCh chan<- bool, tarExcludedGlobs []string) (*pluginclient.ManifestResponse, error) { | ||
| generateManifestStream, err := cmpClient.GenerateManifest(ctx, grpc_retry.Disable()) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error getting generateManifestStream: %s", err) | ||
| } | ||
| opts := []cmp.SenderOption{ | ||
| cmp.WithTarDoneChan(tarDoneCh), | ||
| } | ||
| err = cmp.SendRepoStream(generateManifestStream.Context(), appPath, repoPath, generateManifestStream, env, tarExcludedGlobs, opts...) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error sending file to cmp-server: %s", err) | ||
| } | ||
|
|
@@ -1538,7 +1548,7 @@ func (s *Service) GetAppDetails(ctx context.Context, q *apiclient.RepoServerAppD | |
| return err | ||
| } | ||
|
|
||
| appSourceType, err := GetAppSourceType(ctx, q.Source, opContext.appPath, q.AppName, q.EnabledSourceTypes, s.initConstants.CMPTarExcludedGlobs) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
@@ -1653,7 +1663,7 @@ func loadFileIntoIfExists(path pathutil.ResolvedFilePath, destination *string) e | |
| info, err := os.Stat(stringPath) | ||
|
|
||
| if err == nil && !info.IsDir() { | ||
| bytes, err := ioutil.ReadFile(stringPath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
❤️