diff --git a/cmd/ocp-build-data-enforcer/main.go b/cmd/ocp-build-data-enforcer/main.go index 880c46a2c02..dcf92b529d1 100644 --- a/cmd/ocp-build-data-enforcer/main.go +++ b/cmd/ocp-build-data-enforcer/main.go @@ -6,7 +6,6 @@ import ( "flag" "fmt" "io/ioutil" - "os" "path/filepath" "strings" "sync" @@ -17,291 +16,115 @@ import ( "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" utilerrors "k8s.io/apimachinery/pkg/util/errors" - "sigs.k8s.io/yaml" + git "k8s.io/test-infra/prow/git/v2" + "github.com/openshift/ci-tools/pkg/api/ocpbuilddata" "github.com/openshift/ci-tools/pkg/github" + "github.com/openshift/ci-tools/pkg/github/prcreation" ) type options struct { ocpBuildDataRepoDir string - majorMinor majorMinor + majorMinor ocpbuilddata.MajorMinor + createPRs bool + prCreationCeiling int + *prcreation.PRCreationOptions } -func gatherOptions() *options { - o := &options{} +func gatherOptions() (*options, error) { + o := &options{PRCreationOptions: &prcreation.PRCreationOptions{}} + o.PRCreationOptions.AddFlags(flag.CommandLine) flag.StringVar(&o.ocpBuildDataRepoDir, "ocp-build-data-repo-dir", "../ocp-build-data", "The directory in which the ocp-build-data reposity is") - flag.StringVar(&o.majorMinor.minor, "minor", "6", "The minor version to target") + flag.StringVar(&o.majorMinor.Minor, "minor", "6", "The minor version to target") + flag.BoolVar(&o.createPRs, "create-prs", false, "If the tool should create PRs") + flag.IntVar(&o.prCreationCeiling, "pr-creation-ceiling", 5, "The maximum number of PRs to upsert") flag.Parse() - return o -} -func main() { - opts := gatherOptions() - opts.majorMinor.major = "4" - configsUnverified, err := gatherAllOCPImageConfigs(opts.ocpBuildDataRepoDir, opts.majorMinor) - if err != nil { - logrus.WithError(err).Fatal("Failed to gather all ocp image configs") + if o.createPRs { + if err := o.PRCreationOptions.Finalize(); err != nil { + return nil, fmt.Errorf("failed to finalize pr creation options: %w", err) + } + } else { + o.prCreationCeiling = 0 } + return o, nil +} - streamMap, err := readStreamMap(opts.ocpBuildDataRepoDir, opts.majorMinor) +func main() { + opts, err := gatherOptions() if err != nil { - logrus.WithError(err).Fatal("Failed to read streamMap") + logrus.WithError(err).Fatal("Failed to gather options") } + opts.majorMinor.Major = "4" - groupYAML, err := readGroupYAML(opts.ocpBuildDataRepoDir, opts.majorMinor) + configs, err := ocpbuilddata.LoadImageConfigs(opts.ocpBuildDataRepoDir, opts.majorMinor) if err != nil { - logrus.WithError(err).Fatal("Faild to read groupYAML") - } - - var errs []error - var configs []ocpImageConfig - for _, cfg := range configsUnverified { - if err := cfg.validate(); err != nil { - errs = append(errs, fmt.Errorf("error validating %s: %w", cfg.SourceFileName, err)) - continue - } - if err := dereferenceConfig(&cfg, opts.majorMinor, configsUnverified, streamMap, groupYAML); err != nil { - errs = append(errs, fmt.Errorf("failed dereferencing config for %s: %w", cfg.SourceFileName, err)) - continue - } - configs = append(configs, cfg) - } - - // No point in continuing, that will only result in weird to understand follow-up errors - if err := utilerrors.NewAggregate(errs); err != nil { - for _, err := range err.Errors() { + switch err := err.(type) { + case utilerrors.Aggregate: + for _, err := range err.Errors() { + logrus.WithError(err).Error("Encountered error") + } + default: logrus.WithError(err).Error("Encountered error") } logrus.Fatal("Encountered errors") } + clientFactory, err := git.NewClientFactory() + if err != nil { + logrus.WithError(err).Fatal("Failed to construct git client factory") + } + diffProcessor := dockerfileChangeProcessor(opts.prCreationCeiling, clientFactory, opts.PRCreationOptions) + errGroup := &errgroup.Group{} for idx := range configs { idx := idx errGroup.Go(func() error { - processDockerfile(configs[idx], groupYAML.PublicUpstreams) - return nil + return processDockerfile(configs[idx], diffProcessor) }) } if err := errGroup.Wait(); err != nil { logrus.WithError(err).Fatal("Processing failed") } - if err := utilerrors.NewAggregate(errs); err != nil { - for _, err := range err.Errors() { - logrus.WithError(err).Error("Encountered error") - } - logrus.Fatal("Encountered errors") - } - logrus.Infof("Processed %d configs", len(configs)) -} - -func dereferenceConfig( - config *ocpImageConfig, - majorMinor majorMinor, - allConfigs map[string]ocpImageConfig, - streamMap streamMap, - groupYAML groupYAML, -) error { - var errs []error - - var err error - if config.From.Stream != "" { - config.From.Stream, err = replaceStream(config.From.Stream, streamMap) - if err != nil { - errs = append(errs, fmt.Errorf("failed to replace .from.stream: %w", err)) - } - } - if config.From.Member != "" { - config.From.Stream, err = streamForMember(config.From.Member, majorMinor, allConfigs) - if err != nil { - errs = append(errs, fmt.Errorf("failed to replace .from.member: %w", err)) - } - config.From.Member = "" - } - if config.From.Stream == "" { - errs = append(errs, errors.New("failed to find replacement for .from.stream")) - } - - for blder := range config.From.Builder { - if config.From.Builder[blder].Stream != "" { - config.From.Builder[blder].Stream, err = replaceStream(config.From.Builder[blder].Stream, streamMap) - if err != nil { - errs = append(errs, fmt.Errorf("failed to replace .from[%d].stream: %w", blder, err)) - } - } - if config.From.Builder[blder].Member != "" { - config.From.Builder[blder].Stream, err = streamForMember(config.From.Builder[blder].Member, majorMinor, allConfigs) - if err != nil { - errs = append(errs, fmt.Errorf("failed to replace .from.%d.member: %w", blder, err)) - } - config.From.Builder[blder].Member = "" - } - if config.From.Builder[blder].Stream == "" { - errs = append(errs, fmt.Errorf("failed to dereference from.builder.%d", blder)) - } - } - - if config.Content.Source.Alias != "" { - if _, hasReplacement := groupYAML.Sources[config.Content.Source.Alias]; !hasReplacement { - return fmt.Errorf("groups.yaml has no replacement for alias %s", config.Content.Source.Alias) - } - // Create a new pointer and set its value to groupYAML.Sources[config.Content.Source.Alias] - // rather than directly creating a pointer to the latter. - config.Content.Source.Git = &ocpImageConfigSourceGit{} - *config.Content.Source.Git = groupYAML.Sources[config.Content.Source.Alias] - } - - return utilerrors.NewAggregate(errs) -} - -func streamForMember( - memberName string, - majorMinor majorMinor, - allConfigs map[string]ocpImageConfig, -) (string, error) { - cfgFile := configFileNameForMemberString(memberName) - cfg, cfgExists := allConfigs[cfgFile] - if !cfgExists { - return "", fmt.Errorf("no config %s found", cfgFile) - } - streamTagName := strings.TrimPrefix(cfg.Name, "openshift/ose-") - return fmt.Sprintf("registry.svc.ci.openshift.org/ocp/%s.%s:%s", majorMinor.major, majorMinor.minor, streamTagName), nil -} - -func configFileNameForMemberString(memberString string) string { - return "images/" + memberString + ".yml" + logrus.Infof("Successfully processed %d configs", len(configs)) } -func replaceStream(streamName string, streamMap streamMap) (string, error) { - replacement, hasReplacement := streamMap[streamName] - if !hasReplacement { - return "", fmt.Errorf("streamMap has no replacement for stream %s", streamName) - } - if replacement.UpstreamImage == "" { - return "", fmt.Errorf("stream.yml.%s.upstream_image is an empty string", streamName) - } - return replacement.UpstreamImage, nil -} +type diffProcessor func(l *logrus.Entry, org, repo, path string, oldContent, newContent []byte) error -func processDockerfile(config ocpImageConfig, mappings []publicPrivateMapping) { - orgRepo := config.orgRepo(mappings) - log := logrus.WithField("file", config.SourceFileName).WithField("org/repo", orgRepo) - split := strings.Split(orgRepo, "/") - if n := len(split); n != 2 { - log.Errorf("splitting orgRepo didn't yield 2 but %d results", n) - return - } - if split[0] == "openshift-priv" { +func processDockerfile(config ocpbuilddata.OCPImageConfig, processor diffProcessor) error { + log := logrus.WithField("file", config.SourceFileName).WithField("org/repo", config.PublicRepo.String()) + if config.PublicRepo.Org == "openshift-priv" { log.Trace("Ignoring repo in openshift-priv org") - return + return nil } - org, repo := split[0], split[1] - getter := github.FileGetterFactory(org, repo, "release-4.6") + getter := github.FileGetterFactory(config.PublicRepo.Org, config.PublicRepo.Repo, "release-4.6") - log = log.WithField("dockerfile", config.dockerfile()) - data, err := getter(config.dockerfile()) + log = log.WithField("dockerfile", config.Dockerfile()) + data, err := getter(config.Dockerfile()) if err != nil { - log.WithError(err).Error("Failed to get dockerfile") - return + return fmt.Errorf("failed to get dockerfile: %w", err) } if len(data) == 0 { - log.Error("dockerfile is empty") - return + log.Info("dockerfile is empty") + return nil } updated, hasDiff, err := updateDockerfile(data, config) if err != nil { - log.WithError(err).Error("Failed to update Dockerfile") - return + return fmt.Errorf("failed to update dockerfile: %w", err) } if !hasDiff { - return - } - diff := difflib.UnifiedDiff{ - A: difflib.SplitLines(string(data)), - B: difflib.SplitLines(string(updated)), - FromFile: "original", - ToFile: "updated", - Context: 3, - } - diffStr, err := difflib.GetUnifiedDiffString(diff) - if err != nil { - log.WithError(err).Error("Failed to construct diff") - } - log.Infof("Diff:\n---\n%s\n---\n", diffStr) -} - -func readStreamMap(ocpBuildDataDir string, majorMinor majorMinor) (streamMap, error) { - streamMap := streamMap{} - return streamMap, readYAML(filepath.Join(ocpBuildDataDir, "streams.yml"), &streamMap, majorMinor) -} - -func readGroupYAML(ocpBuildDataDir string, majorMinor majorMinor) (groupYAML, error) { - groupYAML := &groupYAML{} - return *groupYAML, readYAML(filepath.Join(ocpBuildDataDir, "group.yml"), groupYAML, majorMinor) -} - -type majorMinor struct{ major, minor string } - -func readYAML(path string, unmarshalTarget interface{}, majorMinor majorMinor) error { - data, err := ioutil.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read %s: %w", path, err) - } - data = bytes.ReplaceAll(data, []byte("{MAJOR}"), []byte(majorMinor.major)) - data = bytes.ReplaceAll(data, []byte("{MINOR}"), []byte(majorMinor.minor)) - if err := yaml.Unmarshal(data, unmarshalTarget); err != nil { - return fmt.Errorf("unmarshaling failed: %w", err) - } - return nil -} - -func gatherAllOCPImageConfigs(ocpBuildDataDir string, majorMinor majorMinor) (map[string]ocpImageConfig, error) { - result := map[string]ocpImageConfig{} - resultLock := &sync.Mutex{} - errGroup := &errgroup.Group{} - - path := filepath.Join(ocpBuildDataDir, "images") - if err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - errGroup.Go(func() error { - config := &ocpImageConfig{} - if err := readYAML(path, config, majorMinor); err != nil { - return err - } - - // Distgit only repositories - if config.Content == nil { - return nil - } - - config.SourceFileName = strings.TrimPrefix(path, ocpBuildDataDir+"/") - resultLock.Lock() - result[config.SourceFileName] = *config - resultLock.Unlock() - - return nil - }) - return nil - }); err != nil { - return nil, fmt.Errorf("failed to walk") } - - if err := errGroup.Wait(); err != nil { - return nil, fmt.Errorf("failed to read all files: %w", err) + if err := processor(log, config.PublicRepo.Org, config.PublicRepo.Repo, config.Dockerfile(), data, updated); err != nil { + return fmt.Errorf("failed to process updated dockerfile: %w", err) } - return result, nil + return nil } -func updateDockerfile(dockerfile []byte, config ocpImageConfig) ([]byte, bool, error) { +func updateDockerfile(dockerfile []byte, config ocpbuilddata.OCPImageConfig) ([]byte, bool, error) { rootNode, err := imagebuilder.ParseDockerfile(bytes.NewBuffer(dockerfile)) if err != nil { return nil, false, fmt.Errorf("failed to parse Dockerfile: %w", err) @@ -312,7 +135,7 @@ func updateDockerfile(dockerfile []byte, config ocpImageConfig) ([]byte, bool, e return nil, false, fmt.Errorf("failed to construct imagebuilder stages: %w", err) } - cfgStages, err := config.stages() + cfgStages, err := config.Stages() if err != nil { return nil, false, fmt.Errorf("failed to get stages: %w", err) } @@ -380,3 +203,70 @@ type dockerFileReplacment struct { from []byte to []byte } + +func dockerfileChangeProcessor(maxPRs int, gc git.ClientFactory, o *prcreation.PRCreationOptions) diffProcessor { + lock := &sync.Mutex{} + return func(l *logrus.Entry, org, repo, path string, oldContent, newContent []byte) error { + lock.Lock() + defer lock.Unlock() + + // Just print the diff + if maxPRs == 0 { + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines(string(oldContent)), + B: difflib.SplitLines(string(newContent)), + FromFile: "original", + ToFile: "updated", + Context: 3, + } + diffStr, err := difflib.GetUnifiedDiffString(diff) + if err != nil { + return fmt.Errorf("failed to construct diff: %w", err) + } + l.Infof("Diff:\n---\n%s\n---\n", diffStr) + return nil + } + + // Create PR + maxPRs-- + gitClient, err := gc.ClientFor(org, repo) + if err != nil { + return fmt.Errorf("Failed to get git client: %w", err) + } + defer func() { + if err := gitClient.Clean(); err != nil { + l.WithError(err).Error("Gitclient clean failed") + } + }() + + if err := gitClient.Checkout("master"); err != nil { + return fmt.Errorf("failed to checkout master branch: %w", err) + } + if err := ioutil.WriteFile(filepath.Join(gitClient.Directory(), path), newContent, 0644); err != nil { + return fmt.Errorf("failed to write updated Dockerfile into repo: %w", err) + } + if err := o.UpsertPR( + gitClient.Directory(), + org, + repo, + "master", + fmt.Sprintf("Updating %s baseimages to mach ocp-build-data config", path), + strings.Join([]string{ + "This PR is autogenerated by the [ocp-build-data-enforcer][1].", + "It updates the baseimages in the Dockerfile used for promotion in order to ensure it", + "matches the configuration in the [ocp-build-data repository][2] used", + "for producing release artifacts.", + "", + "If you believe the content of this PR is incorrect, please contact the dptp team in", + "#forum-testplatform.", + "", + "[1]: https://github.com/openshift/ci-tools/tree/master/cmd/ocp-build-data-enforcer", + "[2]: https://github.com/openshift/ocp-build-data/tree/openshift-4.6-rhel-8/images", + }, "\n"), + ); err != nil { + return fmt.Errorf("failed to create PR: %w", err) + } + + return nil + } +} diff --git a/cmd/ocp-build-data-enforcer/main_test.go b/cmd/ocp-build-data-enforcer/main_test.go index b529062f722..4cde1432e6b 100644 --- a/cmd/ocp-build-data-enforcer/main_test.go +++ b/cmd/ocp-build-data-enforcer/main_test.go @@ -1,19 +1,18 @@ package main import ( - "errors" - "fmt" "testing" "github.com/google/go-cmp/cmp" - utilerrors "k8s.io/apimachinery/pkg/util/errors" + + "github.com/openshift/ci-tools/pkg/api/ocpbuilddata" ) func TestUpdateDockerfile(t *testing.T) { testCases := []struct { name string dockerfile []byte - config ocpImageConfig + config ocpbuilddata.OCPImageConfig expectededErrMsg string expectedUpdate bool expecteddDockerfile []byte @@ -51,9 +50,9 @@ LABEL io.k8s.display-name="Whereabouts CNI" \ io.k8s.description="This is a component of OpenShift Container Platform and provides a cluster-wide IPAM CNI plugin." \ io.openshift.tags="openshift" \ maintainer="CTO Networking "`), - config: ocpImageConfig{From: ocpImageConfigFrom{ - Builder: []ocpImageConfigFromStream{{Stream: "rhel-8-golang"}, {Stream: "golang"}}, - ocpImageConfigFromStream: ocpImageConfigFromStream{Stream: "rhel"}, + config: ocpbuilddata.OCPImageConfig{From: ocpbuilddata.OCPImageConfigFrom{ + Builder: []ocpbuilddata.OCPImageConfigFromStream{{Stream: "rhel-8-golang"}, {Stream: "golang"}}, + OCPImageConfigFromStream: ocpbuilddata.OCPImageConfigFromStream{Stream: "rhel"}, }}, expecteddDockerfile: []byte(`# This dockerfile is used for building for OpenShift FROM rhel-8-golang as rhel8 @@ -93,9 +92,9 @@ LABEL io.k8s.display-name="Whereabouts CNI" \ FROM openshift/origin-release:rhel-8-golang-1.12 as rhel8 FROM something `), - config: ocpImageConfig{From: ocpImageConfigFrom{ - Builder: []ocpImageConfigFromStream{{Stream: "replaced"}}, - ocpImageConfigFromStream: ocpImageConfigFromStream{Stream: "replacement-2"}, + config: ocpbuilddata.OCPImageConfig{From: ocpbuilddata.OCPImageConfigFrom{ + Builder: []ocpbuilddata.OCPImageConfigFromStream{{Stream: "replaced"}}, + OCPImageConfigFromStream: ocpbuilddata.OCPImageConfigFromStream{Stream: "replacement-2"}, }}, expectedUpdate: true, expecteddDockerfile: []byte(`# This dockerfile is used for building for OpenShift @@ -108,9 +107,9 @@ FROM replacement-2 dockerfile: []byte(`FROM openshift/origin-release:rhel-8-golang-1.12 as rhel8 FROM something `), - config: ocpImageConfig{From: ocpImageConfigFrom{ - Builder: []ocpImageConfigFromStream{{Stream: "replaced"}}, - ocpImageConfigFromStream: ocpImageConfigFromStream{Stream: "replacement-2"}, + config: ocpbuilddata.OCPImageConfig{From: ocpbuilddata.OCPImageConfigFrom{ + Builder: []ocpbuilddata.OCPImageConfigFromStream{{Stream: "replaced"}}, + OCPImageConfigFromStream: ocpbuilddata.OCPImageConfigFromStream{Stream: "replacement-2"}, }}, expectedUpdate: true, expecteddDockerfile: []byte(`FROM replaced as rhel8 @@ -145,130 +144,3 @@ FROM replacement-2 }) } } - -func TestDereferenceConfig(t *testing.T) { - testCases := []struct { - name string - config ocpImageConfig - majorMinor majorMinor - allConfigs map[string]ocpImageConfig - streamMap streamMap - groupYAML groupYAML - expectedConfig ocpImageConfig - expectedError error - }{ - { - name: "config.from.stream gets replaced", - config: ocpImageConfig{ - From: ocpImageConfigFrom{ - ocpImageConfigFromStream: ocpImageConfigFromStream{Stream: "golang"}, - }, - }, - streamMap: streamMap{"golang": {UpstreamImage: "openshift/golang-builder:rhel_8_golang_1.14"}}, - expectedConfig: ocpImageConfig{ - From: ocpImageConfigFrom{ - ocpImageConfigFromStream: ocpImageConfigFromStream{Stream: "openshift/golang-builder:rhel_8_golang_1.14"}, - }, - }, - }, - { - name: "config.from.member gets replaced", - config: ocpImageConfig{ - From: ocpImageConfigFrom{ - ocpImageConfigFromStream: ocpImageConfigFromStream{Member: "openshift-enterprise-base"}, - }, - }, - majorMinor: majorMinor{major: "4", minor: "6"}, - allConfigs: map[string]ocpImageConfig{ - "images/openshift-enterprise-base.yml": {Name: "openshift/ose-base"}, - }, - expectedConfig: ocpImageConfig{ - From: ocpImageConfigFrom{ - ocpImageConfigFromStream: ocpImageConfigFromStream{ - Stream: "registry.svc.ci.openshift.org/ocp/4.6:base"}, - }, - }, - }, - { - name: "both config from.stream and config.from.member are empty, error", - expectedError: errors.New("failed to find replacement for .from.stream"), - }, - { - name: "config.from.builder.stream gets replaced", - config: ocpImageConfig{ - From: ocpImageConfigFrom{ - Builder: []ocpImageConfigFromStream{{Stream: "golang"}}, - ocpImageConfigFromStream: ocpImageConfigFromStream{Stream: "golang"}, - }, - }, - streamMap: streamMap{"golang": {UpstreamImage: "openshift/golang-builder:rhel_8_golang_1.14"}}, - expectedConfig: ocpImageConfig{ - From: ocpImageConfigFrom{ - Builder: []ocpImageConfigFromStream{{Stream: "openshift/golang-builder:rhel_8_golang_1.14"}}, - ocpImageConfigFromStream: ocpImageConfigFromStream{Stream: "openshift/golang-builder:rhel_8_golang_1.14"}, - }, - }, - }, - { - name: "config.from.builder.member gets replaced", - config: ocpImageConfig{ - From: ocpImageConfigFrom{ - Builder: []ocpImageConfigFromStream{{Member: "openshift-enterprise-base"}}, - ocpImageConfigFromStream: ocpImageConfigFromStream{Member: "openshift-enterprise-base"}, - }, - }, - majorMinor: majorMinor{major: "4", minor: "6"}, - allConfigs: map[string]ocpImageConfig{ - "images/openshift-enterprise-base.yml": {Name: "openshift/ose-base"}, - }, - expectedConfig: ocpImageConfig{ - From: ocpImageConfigFrom{ - Builder: []ocpImageConfigFromStream{{Stream: "registry.svc.ci.openshift.org/ocp/4.6:base"}}, - ocpImageConfigFromStream: ocpImageConfigFromStream{ - Stream: "registry.svc.ci.openshift.org/ocp/4.6:base"}, - }, - }, - }, - { - name: "both config.from.builder.stream and config.from.builder.member are empty, error", - config: ocpImageConfig{ - From: ocpImageConfigFrom{ - Builder: []ocpImageConfigFromStream{{}}, - }, - }, - expectedError: utilerrors.NewAggregate([]error{ - errors.New("failed to find replacement for .from.stream"), - fmt.Errorf("failed to dereference from.builder.%d", 0), - }), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if tc.config.Content == nil { - tc.config.Content = &ocpImageConfigContent{} - } - if tc.expectedConfig.Content == nil { - tc.expectedConfig.Content = &ocpImageConfigContent{} - } - var actualErrMsg string - err := dereferenceConfig(&tc.config, tc.majorMinor, tc.allConfigs, tc.streamMap, tc.groupYAML) - if err != nil { - actualErrMsg = err.Error() - } - var expectedErrMsg string - if tc.expectedError != nil { - expectedErrMsg = tc.expectedError.Error() - } - if actualErrMsg != expectedErrMsg { - t.Fatalf("expected error %v, got error %v", tc.expectedError, err) - } - if err != nil { - return - } - if diff := cmp.Diff(tc.config, tc.expectedConfig, cmp.AllowUnexported(ocpImageConfigFrom{})); diff != "" { - t.Errorf("config differs from expectedConfig: %s", diff) - } - }) - } -} diff --git a/cmd/ocp-build-data-enforcer/types.go b/cmd/ocp-build-data-enforcer/types.go deleted file mode 100644 index fdd4347b3be..00000000000 --- a/cmd/ocp-build-data-enforcer/types.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "path/filepath" - "strings" - - utilerrors "k8s.io/apimachinery/pkg/util/errors" -) - -type ocpImageConfig struct { - Content *ocpImageConfigContent `json:"content"` - From ocpImageConfigFrom `json:"from"` - Push ocpImageConfigPush `json:"push"` - Name string `json:"name"` - SourceFileName string `json:"-"` -} - -func (o ocpImageConfig) validate() error { - var errs []error - if o.Content != nil && o.Content.Source.Alias != "" && o.Content.Source.Git != nil { - errs = append(errs, errors.New("both content.source.alias and content.source.git are set")) - } - if err := o.From.validate(); err != nil { - errs = append(errs, fmt.Errorf(".from failed validation: %w", err)) - } - for idx, cfg := range o.From.Builder { - if err := cfg.validate(); err != nil { - errs = append(errs, fmt.Errorf(".from.%d failed validation: %w", idx, err)) - } - } - - return utilerrors.NewAggregate(errs) -} - -type ocpImageConfigContent struct { - Source ocpImageConfigSource `json:"source"` -} - -type ocpImageConfigSource struct { - Dockerfile string `json:"dockerfile"` - Alias string `json:"alias"` - Path string `json:"path"` - // +Optional, mutually exclusive with alias - Git *ocpImageConfigSourceGit `json:"git,omitempty"` -} - -type ocpImageConfigSourceGit struct { - URL string `json:"url"` - Branch ocpImageConfigSourceGitBranch `json:"branch"` -} - -type ocpImageConfigSourceGitBranch struct { - Taget string `json:"target"` -} - -type ocpImageConfigFrom struct { - Builder []ocpImageConfigFromStream `json:"builder"` - ocpImageConfigFromStream `json:",inline"` -} - -type ocpImageConfigFromStream struct { - Stream string `json:"stream"` - Member string `json:"member"` -} - -func (icfs ocpImageConfigFromStream) validate() error { - if icfs.Stream == "" && icfs.Member == "" { - return errors.New("both stream and member were unset") - } - if icfs.Stream != "" && icfs.Member != "" { - return fmt.Errorf("both stream(%s) and member(%s) were set", icfs.Stream, icfs.Member) - } - return nil -} - -type ocpImageConfigPush struct { - Also []string `json:"also,omitempty"` - AdditionalTags []string `json:"additional_tags,omitempty"` -} - -func (oic ocpImageConfig) dockerfile() string { - if oic.Content.Source.Dockerfile == "" { - oic.Content.Source.Dockerfile = "Dockerfile" - } - return filepath.Join(oic.Content.Source.Path, oic.Content.Source.Dockerfile) -} - -func (oic ocpImageConfig) stages() ([]string, error) { - var result []string - var errs []error - for idx, builder := range oic.From.Builder { - if builder.Stream == "" { - errs = append(errs, fmt.Errorf("couldn't dereference from.builder.%d", idx)) - } - result = append(result, builder.Stream) - } - if oic.From.Stream == "" { - errs = append(errs, errors.New("couldn't dereference from.stream")) - } - return append(result, oic.From.Stream), utilerrors.NewAggregate(errs) -} - -func (oic ocpImageConfig) orgRepo(mappings []publicPrivateMapping) string { - var name string - if oic.Content.Source.Git.URL == "" { - name = oic.Name - } - if name == "" { - name = strings.TrimSuffix(strings.TrimPrefix(oic.Content.Source.Git.URL, "git@github.com:"), ".git") - } - return getPublicRepo(name, mappings) -} - -type streamMap map[string]streamElement - -type streamElement struct { - Image string `json:"image"` - UpstreamImage string `json:"upstream_image"` -} - -type groupYAML struct { - Sources map[string]ocpImageConfigSourceGit `json:"sources"` - PublicUpstreams []publicPrivateMapping `json:"public_upstreams,omitempty"` -} - -type publicPrivateMapping struct { - Private string `json:"private"` - Public string `json:"public"` -} - -func getPublicRepo(orgRepo string, mappings []publicPrivateMapping) string { - orgRepo = "https://github.com/" + orgRepo - var replacementFrom, replacementTo string - for _, mapping := range mappings { - if !strings.HasPrefix(orgRepo, mapping.Private) { - continue - } - if len(replacementFrom) > len(mapping.Private) { - continue - } - replacementFrom = mapping.Private - replacementTo = mapping.Public - } - - if replacementTo == "" { - return strings.TrimPrefix(orgRepo, "https://github.com/") - } - - return strings.TrimPrefix(strings.Replace(orgRepo, replacementFrom, replacementTo, 1), "https://github.com/") -} diff --git a/cmd/ocp-build-data-enforcer/types_test.go b/cmd/ocp-build-data-enforcer/types_test.go deleted file mode 100644 index f38666d5dd1..00000000000 --- a/cmd/ocp-build-data-enforcer/types_test.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "testing" - - "sigs.k8s.io/yaml" -) - -func TestOCPImageConfig(t *testing.T) { - testcases := []struct { - name string - in []byte - expectedGitURL string - expectedStream string - expectedName string - }{ - { - name: "simple", - in: []byte(`content: - source: - dockerfile: Dockerfile.openshift - git: - branch: - target: release-{MAJOR}.{MINOR} - url: git@github.com:openshift/whereabouts-cni.git -distgit: - branch: rhaos-{MAJOR}.{MINOR}-rhel-7 -from: - builder: - - stream: rhel-8-golang - - stream: golang - stream: rhel -name: openshift/ose-multus-whereabouts-ipam-cni -owners: -- multus-dev@redhat.com`), - expectedGitURL: "git@github.com:openshift/whereabouts-cni.git", - expectedStream: "rhel", - expectedName: "openshift/ose-multus-whereabouts-ipam-cni", - }, - { - name: "complex", - in: []byte(`container_yaml: - go: - modules: - - module: k8s.io/autoscaler -content: - source: - dockerfile: images/cluster-autoscaler/Dockerfile.rhel7 - git: - branch: - target: release-{MAJOR}.{MINOR} - url: git@github.com:openshift-priv/kubernetes-autoscaler.git -distgit: - namespace: containers -enabled_repos: -- rhel-8-baseos-rpms -- rhel-8-appstream-rpms -- rhel-8-server-ose-rpms -from: - builder: - - stream: golang - stream: rhel -labels: - License: ASL 2.0 - io.k8s.description: Cluster Autoscaler for OpenShift and Kubernetes. - io.k8s.display-name: OpenShift Container Platform Cluster Autoscaler - io.openshift.tags: openshift,autoscaler - vendor: Red Hat -name: openshift/ose-cluster-autoscaler -owners: -- avagarwa@redhat.com -`), - expectedGitURL: "git@github.com:openshift-priv/kubernetes-autoscaler.git", - expectedStream: "rhel", - expectedName: "openshift/ose-cluster-autoscaler", - }, - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - ocpImageConfig := &ocpImageConfig{} - if err := yaml.Unmarshal(tc.in, ocpImageConfig); err != nil { - t.Fatalf("unmarshal failed: %v", err) - } - - if ocpImageConfig.Content.Source.Git.URL != tc.expectedGitURL { - t.Errorf("expected ocpImageConfig.Content.Source.Git.URL to be %s, was %s", tc.expectedGitURL, ocpImageConfig.Content.Source.Git.URL) - } - - if ocpImageConfig.From.Stream != tc.expectedStream { - t.Errorf("expected ocpImageConfig.From.Stream to be %s, was %s", tc.expectedStream, ocpImageConfig.From.Stream) - } - if ocpImageConfig.Name != tc.expectedName { - t.Errorf("expected name to be %s, was %s", tc.expectedName, ocpImageConfig.Name) - } - }) - - } -} - -func TestGetPublicRepo(t *testing.T) { - testCases := []struct { - name string - orgRepoIn string - mappings []publicPrivateMapping - - expected string - }{ - { - name: "no match, original string is returned", - orgRepoIn: "kubeflow/kubeflow", - mappings: []publicPrivateMapping{ - {Private: "https://github.com/openshift-priv", Public: "https://github.com/openshift"}, - {Private: "https://github.com/openshift/ose", Public: "https://github.com/openshift/origin"}, - }, - expected: "kubeflow/kubeflow", - }, - { - name: "single match is used", - orgRepoIn: "kubeflow-priv/kubeflow", - mappings: []publicPrivateMapping{ - {Private: "https://github.com/kubeflow-priv", Public: "https://github.com/kubeflow"}, - {Private: "https://github.com/openshift/ose", Public: "https://github.com/openshift/origin"}, - }, - expected: "kubeflow/kubeflow", - }, - { - name: "multiple matches, longest prefix match is used", - orgRepoIn: "kubeflow-priv/kubeflow", - mappings: []publicPrivateMapping{ - {Private: "https://github.com/kubeflow-priv", Public: "https://github.com/kubeflow"}, - {Private: "https://github.com/kubeflow-priv/kubeflow", Public: "https://github.com/openshift/origin"}, - }, - expected: "openshift/origin", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if actual := getPublicRepo(tc.orgRepoIn, tc.mappings); actual != tc.expected { - t.Errorf("expected %s, got %s", tc.expected, actual) - } - }) - } -} diff --git a/pkg/api/ocpbuilddata/types.go b/pkg/api/ocpbuilddata/types.go new file mode 100644 index 00000000000..f63f06f3f7d --- /dev/null +++ b/pkg/api/ocpbuilddata/types.go @@ -0,0 +1,371 @@ +package ocpbuilddata + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "sigs.k8s.io/yaml" +) + +type OCPImageConfig struct { + Content *OCPImageConfigContent `json:"content"` + From OCPImageConfigFrom `json:"from"` + Push OCPImageConfigPush `json:"push"` + Name string `json:"name"` + SourceFileName string `json:"-"` + Version MajorMinor `json:"-"` + PublicRepo OrgRepo `json:"-"` +} + +func (o OCPImageConfig) validate() error { + var errs []error + if o.Content != nil && o.Content.Source.Alias != "" && o.Content.Source.Git != nil { + errs = append(errs, errors.New("both content.source.alias and content.source.git are set")) + } + if err := o.From.validate(); err != nil { + errs = append(errs, fmt.Errorf(".from failed validation: %w", err)) + } + for idx, cfg := range o.From.Builder { + if err := cfg.validate(); err != nil { + errs = append(errs, fmt.Errorf(".from.%d failed validation: %w", idx, err)) + } + } + + return utilerrors.NewAggregate(errs) +} + +func (o OCPImageConfig) PromotesTo() string { + return fmt.Sprintf("registry.svc.ci.openshift.org/ocp/%s.%s:%s", o.Version.Major, o.Version.Minor, strings.TrimPrefix(o.Name, "openshift/ose-")) +} + +type OCPImageConfigContent struct { + Source OCPImageConfigSource `json:"source"` +} + +type OrgRepo struct { + Org string + Repo string +} + +func (o OrgRepo) String() string { + return o.Org + "/" + o.Repo +} + +type OCPImageConfigSource struct { + Dockerfile string `json:"dockerfile"` + Alias string `json:"alias"` + Path string `json:"path"` + // +Optional, mutually exclusive with alias + Git *OCPImageConfigSourceGit `json:"git,omitempty"` +} + +type OCPImageConfigSourceGit struct { + URL string `json:"url"` + Branch OCPImageConfigSourceGitBRanch `json:"branch"` +} + +type OCPImageConfigSourceGitBRanch struct { + Taget string `json:"target"` +} + +type OCPImageConfigFrom struct { + Builder []OCPImageConfigFromStream `json:"builder"` + OCPImageConfigFromStream `json:",inline"` +} + +type OCPImageConfigFromStream struct { + Stream string `json:"stream"` + Member string `json:"member"` +} + +func (icfs OCPImageConfigFromStream) validate() error { + if icfs.Stream == "" && icfs.Member == "" { + return errors.New("both stream and member were unset") + } + if icfs.Stream != "" && icfs.Member != "" { + return fmt.Errorf("both stream(%s) and member(%s) were set", icfs.Stream, icfs.Member) + } + return nil +} + +type OCPImageConfigPush struct { + Also []string `json:"also,omitempty"` + AdditionalTags []string `json:"additional_tags,omitempty"` +} + +func (oic *OCPImageConfig) Dockerfile() string { + if oic.Content.Source.Dockerfile == "" { + oic.Content.Source.Dockerfile = "Dockerfile" + } + return filepath.Join(oic.Content.Source.Path, oic.Content.Source.Dockerfile) +} + +func (oic *OCPImageConfig) Stages() ([]string, error) { + var result []string + var errs []error + for idx, builder := range oic.From.Builder { + if builder.Stream == "" { + errs = append(errs, fmt.Errorf("couldn't dereference from.builder.%d", idx)) + } + result = append(result, builder.Stream) + } + if oic.From.Stream == "" { + errs = append(errs, errors.New("couldn't dereference from.stream")) + } + return append(result, oic.From.Stream), utilerrors.NewAggregate(errs) +} + +func (oic *OCPImageConfig) setPublicOrgRepo(mappings []PublicPrivateMapping) { + var name string + if oic.Content == nil || oic.Content.Source.Git == nil || oic.Content.Source.Git.URL == "" { + name = oic.Name + } + if name == "" && oic.Content != nil && oic.Content.Source.Git != nil { + name = strings.TrimSuffix(strings.TrimPrefix(oic.Content.Source.Git.URL, "git@github.com:"), ".git") + } + + oic.PublicRepo.Org = publicRepo(name, mappings) + if split := strings.Split(oic.PublicRepo.Org, "/"); len(split) == 2 { + oic.PublicRepo.Org = split[0] + oic.PublicRepo.Repo = split[1] + } +} + +type StreamMap map[string]StreamElement + +type StreamElement struct { + Image string `json:"image"` + UpstreamImage string `json:"upstream_image"` +} + +type GroupYAML struct { + Sources map[string]OCPImageConfigSourceGit `json:"sources"` + PublicUpstreams []PublicPrivateMapping `json:"public_upstreams,omitempty"` +} + +type PublicPrivateMapping struct { + Private string `json:"private"` + Public string `json:"public"` +} + +func publicRepo(orgRepo string, mappings []PublicPrivateMapping) string { + orgRepo = "https://github.com/" + orgRepo + var replacementFrom, replacementTo string + for _, mapping := range mappings { + if !strings.HasPrefix(orgRepo, mapping.Private) { + continue + } + if len(replacementFrom) > len(mapping.Private) { + continue + } + replacementFrom = mapping.Private + replacementTo = mapping.Public + } + + if replacementTo == "" { + return strings.TrimPrefix(orgRepo, "https://github.com/") + } + + return strings.TrimPrefix(strings.Replace(orgRepo, replacementFrom, replacementTo, 1), "https://github.com/") +} + +// LoadImageConfigs loads and dereferences all image configs from the provided ocp-build-data repo root +func LoadImageConfigs(ocpBuildDataDir string, majorMinor MajorMinor) ([]OCPImageConfig, error) { + configsUnverified, err := gatherAllOCPImageConfigs(ocpBuildDataDir, majorMinor) + if err != nil { + return nil, fmt.Errorf("failed to read all image configs: %w", err) + } + streamMap, err := readStreamMap(ocpBuildDataDir, majorMinor) + if err != nil { + return nil, fmt.Errorf("failed to read streams file: %w", err) + } + + groupYAML, err := readGroupYAML(ocpBuildDataDir, majorMinor) + if err != nil { + return nil, fmt.Errorf("failed to read group file: %w", err) + } + + var errs []error + var configs []OCPImageConfig + for _, cfg := range configsUnverified { + if err := cfg.validate(); err != nil { + errs = append(errs, fmt.Errorf("error validating %s: %w", cfg.SourceFileName, err)) + continue + } + if err := dereferenceConfig(&cfg, configsUnverified, streamMap, groupYAML); err != nil { + errs = append(errs, fmt.Errorf("failed dereferencing config for %s: %w", cfg.SourceFileName, err)) + continue + } + configs = append(configs, cfg) + } + + return configs, utilerrors.NewAggregate(errs) +} + +func dereferenceConfig( + config *OCPImageConfig, + allConfigs map[string]OCPImageConfig, + streamMap StreamMap, + groupYAML GroupYAML, +) error { + var errs []error + + var err error + if config.From.Stream != "" { + config.From.Stream, err = replaceStream(config.From.Stream, streamMap) + if err != nil { + errs = append(errs, fmt.Errorf("failed to replace .from.stream: %w", err)) + } + } + if config.From.Member != "" { + config.From.Stream, err = streamForMember(config.From.Member, allConfigs) + if err != nil { + errs = append(errs, fmt.Errorf("failed to replace .from.member: %w", err)) + } + config.From.Member = "" + } + if config.From.Stream == "" { + errs = append(errs, errors.New("failed to find replacement for .from.stream")) + } + + for blder := range config.From.Builder { + if config.From.Builder[blder].Stream != "" { + config.From.Builder[blder].Stream, err = replaceStream(config.From.Builder[blder].Stream, streamMap) + if err != nil { + errs = append(errs, fmt.Errorf("failed to replace .from.%d.stream: %w", blder, err)) + } + } + if config.From.Builder[blder].Member != "" { + config.From.Builder[blder].Stream, err = streamForMember(config.From.Builder[blder].Member, allConfigs) + if err != nil { + errs = append(errs, fmt.Errorf("failed to replace .from.%d.member: %w", blder, err)) + } + config.From.Builder[blder].Member = "" + } + if config.From.Builder[blder].Stream == "" { + errs = append(errs, fmt.Errorf("failed to dereference from.builder.%d", blder)) + } + } + + if config.Content.Source.Alias != "" { + if _, hasReplacement := groupYAML.Sources[config.Content.Source.Alias]; !hasReplacement { + return fmt.Errorf("groups.yaml has no replacement for alias %s", config.Content.Source.Alias) + } + // Create a new pointer and set its value to groupYAML.Sources[config.Content.Source.Alias] + // rather than directly creating a pointer to the latter. + config.Content.Source.Git = &OCPImageConfigSourceGit{} + *config.Content.Source.Git = groupYAML.Sources[config.Content.Source.Alias] + } + + config.setPublicOrgRepo(groupYAML.PublicUpstreams) + + return utilerrors.NewAggregate(errs) +} + +func replaceStream(streamName string, streamMap StreamMap) (string, error) { + replacement, hasReplacement := streamMap[streamName] + if !hasReplacement { + return "", fmt.Errorf("streamMap has no replacement for stream %s", streamName) + } + if replacement.UpstreamImage == "" { + return "", fmt.Errorf("stream.yml.%s.upstream_image is an empty string", streamName) + } + return replacement.UpstreamImage, nil +} + +func configFileNamberForMemberString(memberString string) string { + return "images/" + memberString + ".yml" +} + +func streamForMember( + memberName string, + allConfigs map[string]OCPImageConfig, +) (string, error) { + cfgFile := configFileNamberForMemberString(memberName) + cfg, cfgExists := allConfigs[cfgFile] + if !cfgExists { + return "", fmt.Errorf("no config %s found", cfgFile) + } + return cfg.PromotesTo(), nil +} + +func readStreamMap(ocpBuildDataDir string, majorMinor MajorMinor) (StreamMap, error) { + streamMap := StreamMap{} + return streamMap, readYAML(filepath.Join(ocpBuildDataDir, "streams.yml"), &streamMap, majorMinor) +} + +func readGroupYAML(ocpBuildDataDir string, majorMinor MajorMinor) (GroupYAML, error) { + groupYAML := GroupYAML{} + return groupYAML, readYAML(filepath.Join(ocpBuildDataDir, "group.yml"), &groupYAML, majorMinor) +} + +type MajorMinor struct { + Major string + Minor string +} + +func gatherAllOCPImageConfigs(ocpBuildDataDir string, majorMinor MajorMinor) (map[string]OCPImageConfig, error) { + result := map[string]OCPImageConfig{} + resultLock := &sync.Mutex{} + errGroup := &errgroup.Group{} + + path := filepath.Join(ocpBuildDataDir, "images") + if err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + errGroup.Go(func() error { + config := OCPImageConfig{} + if err := readYAML(path, &config, majorMinor); err != nil { + return err + } + + // Distgit only repositories + if config.Content == nil { + return nil + } + + config.SourceFileName = strings.TrimPrefix(path, ocpBuildDataDir+"/") + config.Version = majorMinor + resultLock.Lock() + result[config.SourceFileName] = config + resultLock.Unlock() + + return nil + }) + + return nil + }); err != nil { + return nil, fmt.Errorf("failed to walk") + } + + if err := errGroup.Wait(); err != nil { + return nil, fmt.Errorf("failed to read all files: %w", err) + } + + return result, nil +} + +func readYAML(path string, unmarshalTarget interface{}, majorMinor MajorMinor) error { + data, err := ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read %s: %w", path, err) + } + data = bytes.ReplaceAll(data, []byte("{MAJOR}"), []byte(majorMinor.Major)) + data = bytes.ReplaceAll(data, []byte("{MINOR}"), []byte(majorMinor.Minor)) + if err := yaml.Unmarshal(data, unmarshalTarget); err != nil { + return fmt.Errorf("unmarshaling failed: %w", err) + } + return nil +} diff --git a/pkg/api/ocpbuilddata/types_test.go b/pkg/api/ocpbuilddata/types_test.go new file mode 100644 index 00000000000..fc3f93bf87a --- /dev/null +++ b/pkg/api/ocpbuilddata/types_test.go @@ -0,0 +1,288 @@ +package ocpbuilddata + +import ( + "errors" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "sigs.k8s.io/yaml" +) + +func TestOCPImageConfig(t *testing.T) { + testcases := []struct { + name string + in []byte + expectedGitURL string + expectedStream string + expectedName string + }{ + { + name: "simple", + in: []byte(`content: + source: + dockerfile: Dockerfile.openshift + git: + branch: + target: release-{MAJOR}.{MINOR} + url: git@github.com:openshift/whereabouts-cni.git +distgit: + branch: rhaos-{MAJOR}.{MINOR}-rhel-7 +from: + builder: + - stream: rhel-8-golang + - stream: golang + stream: rhel +name: openshift/ose-multus-whereabouts-ipam-cni +owners: +- multus-dev@redhat.com`), + expectedGitURL: "git@github.com:openshift/whereabouts-cni.git", + expectedStream: "rhel", + expectedName: "openshift/ose-multus-whereabouts-ipam-cni", + }, + { + name: "complex", + in: []byte(`container_yaml: + go: + modules: + - module: k8s.io/autoscaler +content: + source: + dockerfile: images/cluster-autoscaler/Dockerfile.rhel7 + git: + branch: + target: release-{MAJOR}.{MINOR} + url: git@github.com:openshift-priv/kubernetes-autoscaler.git +distgit: + namespace: containers +enabled_repos: +- rhel-8-baseos-rpms +- rhel-8-appstream-rpms +- rhel-8-server-ose-rpms +from: + builder: + - stream: golang + stream: rhel +labels: + License: ASL 2.0 + io.k8s.description: Cluster Autoscaler for OpenShift and Kubernetes. + io.k8s.display-name: OpenShift Container Platform Cluster Autoscaler + io.openshift.tags: openshift,autoscaler + vendor: Red Hat +name: openshift/ose-cluster-autoscaler +owners: +- avagarwa@redhat.com +`), + expectedGitURL: "git@github.com:openshift-priv/kubernetes-autoscaler.git", + expectedStream: "rhel", + expectedName: "openshift/ose-cluster-autoscaler", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + ocpImageConfig := &OCPImageConfig{} + if err := yaml.Unmarshal(tc.in, ocpImageConfig); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + + if ocpImageConfig.Content.Source.Git.URL != tc.expectedGitURL { + t.Errorf("expected ocpImageConfig.Content.Source.Git.URL to be %s, was %s", tc.expectedGitURL, ocpImageConfig.Content.Source.Git.URL) + } + + if ocpImageConfig.From.Stream != tc.expectedStream { + t.Errorf("expected ocpImageConfig.From.Stream to be %s, was %s", tc.expectedStream, ocpImageConfig.From.Stream) + } + if ocpImageConfig.Name != tc.expectedName { + t.Errorf("expected name to be %s, was %s", tc.expectedName, ocpImageConfig.Name) + } + }) + + } +} + +func TestSetPublicRepo(t *testing.T) { + testCases := []struct { + name string + orgRepoIn string + mappings []PublicPrivateMapping + + expected OrgRepo + }{ + { + name: "no match, original string is returned", + orgRepoIn: "kubeflow/kubeflow", + mappings: []PublicPrivateMapping{ + {Private: "https://github.com/openshift-priv", Public: "https://github.com/openshift"}, + {Private: "https://github.com/openshift/ose", Public: "https://github.com/openshift/origin"}, + }, + expected: OrgRepo{Org: "kubeflow", Repo: "kubeflow"}, + }, + { + name: "single match is used", + orgRepoIn: "kubeflow-priv/kubeflow", + mappings: []PublicPrivateMapping{ + {Private: "https://github.com/kubeflow-priv", Public: "https://github.com/kubeflow"}, + {Private: "https://github.com/openshift/ose", Public: "https://github.com/openshift/origin"}, + }, + expected: OrgRepo{Org: "kubeflow", Repo: "kubeflow"}, + }, + { + name: "multiple matches, longest prefix match is used", + orgRepoIn: "kubeflow-priv/kubeflow", + mappings: []PublicPrivateMapping{ + {Private: "https://github.com/kubeflow-priv", Public: "https://github.com/kubeflow"}, + {Private: "https://github.com/kubeflow-priv/kubeflow", Public: "https://github.com/openshift/origin"}, + }, + expected: OrgRepo{Org: "openshift", Repo: "origin"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cfg := &OCPImageConfig{Name: tc.orgRepoIn} + cfg.setPublicOrgRepo(tc.mappings) + if diff := cmp.Diff(cfg.PublicRepo, tc.expected); diff != "" { + t.Errorf("actual differs from expected: %s", diff) + } + }) + } +} + +func TestDereferenceConfig(t *testing.T) { + testCases := []struct { + name string + config OCPImageConfig + majorMinor MajorMinor + allConfigs map[string]OCPImageConfig + streamMap StreamMap + groupYAML GroupYAML + expectedConfig OCPImageConfig + expectedError error + }{ + { + name: "config.from.stream gets replaced", + config: OCPImageConfig{ + From: OCPImageConfigFrom{ + OCPImageConfigFromStream: OCPImageConfigFromStream{Stream: "golang"}, + }, + }, + streamMap: StreamMap{"golang": {UpstreamImage: "openshift/golang-builder:rhel_8_golang_1.14"}}, + expectedConfig: OCPImageConfig{ + From: OCPImageConfigFrom{ + OCPImageConfigFromStream: OCPImageConfigFromStream{Stream: "openshift/golang-builder:rhel_8_golang_1.14"}, + }, + }, + }, + { + name: "config.from.member gets replaced", + config: OCPImageConfig{ + From: OCPImageConfigFrom{ + OCPImageConfigFromStream: OCPImageConfigFromStream{Member: "openshift-enterprise-base"}, + }, + Version: MajorMinor{Major: "4", Minor: "6"}, + }, + allConfigs: map[string]OCPImageConfig{ + "images/openshift-enterprise-base.yml": { + Name: "openshift/ose-base", + Version: MajorMinor{Major: "4", Minor: "6"}, + }, + }, + expectedConfig: OCPImageConfig{ + From: OCPImageConfigFrom{ + OCPImageConfigFromStream: OCPImageConfigFromStream{ + Stream: "registry.svc.ci.openshift.org/ocp/4.6:base"}, + }, + }, + }, + { + name: "both config from.stream and config.from.member are empty, error", + expectedError: errors.New("failed to find replacement for .from.stream"), + }, + { + name: "config.from.builder.stream gets replaced", + config: OCPImageConfig{ + From: OCPImageConfigFrom{ + Builder: []OCPImageConfigFromStream{{Stream: "golang"}}, + OCPImageConfigFromStream: OCPImageConfigFromStream{Stream: "golang"}, + }, + }, + streamMap: StreamMap{"golang": {UpstreamImage: "openshift/golang-builder:rhel_8_golang_1.14"}}, + expectedConfig: OCPImageConfig{ + From: OCPImageConfigFrom{ + Builder: []OCPImageConfigFromStream{{Stream: "openshift/golang-builder:rhel_8_golang_1.14"}}, + OCPImageConfigFromStream: OCPImageConfigFromStream{Stream: "openshift/golang-builder:rhel_8_golang_1.14"}, + }, + }, + }, + { + name: "config.from.builder.member gets replaced", + config: OCPImageConfig{ + From: OCPImageConfigFrom{ + Builder: []OCPImageConfigFromStream{{Member: "openshift-enterprise-base"}}, + OCPImageConfigFromStream: OCPImageConfigFromStream{Member: "openshift-enterprise-base"}, + }, + Version: MajorMinor{Major: "4", Minor: "6"}, + }, + allConfigs: map[string]OCPImageConfig{ + "images/openshift-enterprise-base.yml": { + Name: "openshift/ose-base", + Version: MajorMinor{Major: "4", Minor: "6"}, + }, + }, + expectedConfig: OCPImageConfig{ + From: OCPImageConfigFrom{ + Builder: []OCPImageConfigFromStream{{Stream: "registry.svc.ci.openshift.org/ocp/4.6:base"}}, + OCPImageConfigFromStream: OCPImageConfigFromStream{ + Stream: "registry.svc.ci.openshift.org/ocp/4.6:base"}, + }, + }, + }, + { + name: "both config.from.builder.stream and config.from.builder.member are empty, error", + config: OCPImageConfig{ + From: OCPImageConfigFrom{ + Builder: []OCPImageConfigFromStream{{}}, + }, + }, + expectedError: utilerrors.NewAggregate([]error{ + errors.New("failed to find replacement for .from.stream"), + fmt.Errorf("failed to dereference from.builder.%d", 0), + }), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.config.Version.Major = "4" + tc.config.Version.Minor = "6" + tc.expectedConfig.Version.Major = "4" + tc.expectedConfig.Version.Minor = "6" + if tc.config.Content == nil { + tc.config.Content = &OCPImageConfigContent{} + } + if tc.expectedConfig.Content == nil { + tc.expectedConfig.Content = &OCPImageConfigContent{} + } + var actualErrMsg string + err := dereferenceConfig(&tc.config, tc.allConfigs, tc.streamMap, tc.groupYAML) + if err != nil { + actualErrMsg = err.Error() + } + var expectedErrMsg string + if tc.expectedError != nil { + expectedErrMsg = tc.expectedError.Error() + } + if actualErrMsg != expectedErrMsg { + t.Fatalf("expected error %v, got error %v", tc.expectedError, err) + } + if err != nil { + return + } + if diff := cmp.Diff(tc.config, tc.expectedConfig); diff != "" { + t.Errorf("config differs from expectedConfig: %s", diff) + } + }) + } +} diff --git a/pkg/github/prcreation/prcreation.go b/pkg/github/prcreation/prcreation.go new file mode 100644 index 00000000000..25e9078e15f --- /dev/null +++ b/pkg/github/prcreation/prcreation.go @@ -0,0 +1,106 @@ +package prcreation + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/sirupsen/logrus" + "k8s.io/test-infra/experiment/autobumper/bumper" + "k8s.io/test-infra/prow/config/secret" + "k8s.io/test-infra/prow/flagutil" + "k8s.io/test-infra/prow/github" + "k8s.io/test-infra/prow/labels" +) + +type PRCreationOptions struct { + SelfApprove bool + flagutil.GitHubOptions + secretAgent *secret.Agent + GithubClient github.Client +} + +func (o *PRCreationOptions) AddFlags(fs *flag.FlagSet) { + o.GitHubOptions.AddFlags(fs) +} + +func (o *PRCreationOptions) Finalize() error { + if err := o.GitHubOptions.Validate(false); err != nil { + return err + } + o.secretAgent = &secret.Agent{} + if err := o.secretAgent.Start([]string{o.TokenPath}); err != nil { + return fmt.Errorf("failed to start secretAgent: %w", err) + } + var err error + o.GithubClient, err = o.GitHubClient(o.secretAgent, false) + if err != nil { + return fmt.Errorf("failed to construct github client: %w", err) + } + + return nil +} + +// UpsertPR upserts a PR. The PRTitle must be alphanumeric except for spaces, as it will be used as the +// branchname on the bots fork. +func (o *PRCreationOptions) UpsertPR(localSourceDir, org, repo, branch, prTitle, prBody string) error { + if err := os.Chdir(localSourceDir); err != nil { + return fmt.Errorf("failed to chdir into %s: %w", localSourceDir, err) + } + + changed, err := bumper.HasChanges() + if err != nil { + return fmt.Errorf("failed to check for changes: %w", err) + } + + if !changed { + logrus.Info("No changes, not upserting PR") + return nil + } + + username, err := o.GithubClient.BotName() + if err != nil { + return fmt.Errorf("failed to get botname: %w", err) + } + token := o.secretAgent.GetSecret(o.TokenPath) + stdout := bumper.HideSecretsWriter{Delegate: os.Stdout, Censor: o.secretAgent} + stderr := bumper.HideSecretsWriter{Delegate: os.Stderr, Censor: o.secretAgent} + + sourceBranchName := strings.ReplaceAll(strings.ToLower(prTitle), " ", "-") + + if err := bumper.GitCommitAndPush( + fmt.Sprintf("https://%s:%s@github.com/%s/%s.git", username, string(token), org, repo), + sourceBranchName, + username, + fmt.Sprintf("%s@users.noreply.github.com", username), + prTitle+"\n\n"+prBody, + stdout, + stderr, + ); err != nil { + return fmt.Errorf("failed to push changes: %w", err) + } + + var labelsToAdd []string + if o.SelfApprove { + logrus.Infof("Self-aproving PR by adding the %q and %q labels", labels.Approved, labels.LGTM) + labelsToAdd = append(labelsToAdd, labels.Approved, labels.LGTM) + } + + if err := bumper.UpdatePullRequestWithLabels( + o.GithubClient, + org, + repo, + prTitle, + prBody, + prTitle, + username+":"+sourceBranchName, + branch, + true, + labelsToAdd, + ); err != nil { + return fmt.Errorf("failed to upsert PR: %w", err) + } + + return nil +}