From 3571013122a45d1ab590e4512fbec641052a62d0 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Tue, 28 Aug 2018 14:13:47 +0200 Subject: [PATCH] Add a simple program to make Prow job config ordering deterministic --- cmd/ci-operator-prowgen/main.go | 48 ++-------------- cmd/determinize-prow-jobs/main.go | 74 +++++++++++++++++++++++++ images/determinize-prow-jobs/Dockerfile | 5 ++ pkg/jobconfig/files.go | 41 ++++++++++++++ 4 files changed, 124 insertions(+), 44 deletions(-) create mode 100644 cmd/determinize-prow-jobs/main.go create mode 100644 images/determinize-prow-jobs/Dockerfile create mode 100644 pkg/jobconfig/files.go diff --git a/cmd/ci-operator-prowgen/main.go b/cmd/ci-operator-prowgen/main.go index 1e48d236..4bad351c 100644 --- a/cmd/ci-operator-prowgen/main.go +++ b/cmd/ci-operator-prowgen/main.go @@ -10,12 +10,12 @@ import ( "path/filepath" "strings" - "github.com/ghodss/yaml" - cioperatorapi "github.com/openshift/ci-operator/pkg/api" kubeapi "k8s.io/api/core/v1" prowconfig "k8s.io/test-infra/prow/config" prowkube "k8s.io/test-infra/prow/kube" + + jc "github.com/openshift/ci-operator-prowgen/pkg/jobconfig" ) type options struct { @@ -304,46 +304,6 @@ func generateJobsFromDirectory(configDir, jobDir, jobFile string) error { return nil } -func readJobConfig(path string) (*prowconfig.JobConfig, error) { - data, err := ioutil.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read Prow job config (%v)", err) - } - - var jobConfig *prowconfig.JobConfig - if err := yaml.Unmarshal(data, &jobConfig); err != nil { - return nil, fmt.Errorf("failed to load Prow job config (%v)", err) - } - if jobConfig == nil { // happens when `data` is empty - return nil, fmt.Errorf("failed to load Prow job config") - } - - return jobConfig, nil -} - -// Print JobConfig to stdout as YAML -func writeJobs(jobConfig *prowconfig.JobConfig) error { - jobConfigAsYaml, err := yaml.Marshal(*jobConfig) - if err != nil { - return fmt.Errorf("failed to marshal the job config (%v)", err) - } - fmt.Printf(string(jobConfigAsYaml)) - return nil -} - -// Write JobConfig to a file as YAML -func writeJobsToFile(path string, jobConfig *prowconfig.JobConfig) error { - jobConfigAsYaml, err := yaml.Marshal(*jobConfig) - if err != nil { - return fmt.Errorf("failed to marshal the job config (%v)", err) - } - if err := ioutil.WriteFile(path, jobConfigAsYaml, 0664); err != nil { - return fmt.Errorf("Failed to write job config to '%s' (%v)", path, err) - } - - return nil -} - // Given two JobConfig, merge jobs from the `source` one to to `destination` // one. Jobs are matched by name. All jobs from `source` will be present in // `destination` - if there were jobs with the same name in `destination`, they @@ -401,14 +361,14 @@ func mergeJobConfig(destination, source *prowconfig.JobConfig) { // to the file path. If the file already contains some jobs, new ones will be // merged with the existing ones. func mergeJobsIntoFile(prowConfigPath string, jobConfig *prowconfig.JobConfig) error { - existingJobConfig, err := readJobConfig(prowConfigPath) + existingJobConfig, err := jc.ReadFromFile(prowConfigPath) if err != nil { existingJobConfig = &prowconfig.JobConfig{} } mergeJobConfig(existingJobConfig, jobConfig) - if err = writeJobsToFile(prowConfigPath, existingJobConfig); err != nil { + if err = jc.WriteToFile(prowConfigPath, existingJobConfig); err != nil { return err } diff --git a/cmd/determinize-prow-jobs/main.go b/cmd/determinize-prow-jobs/main.go new file mode 100644 index 00000000..f0f18afa --- /dev/null +++ b/cmd/determinize-prow-jobs/main.go @@ -0,0 +1,74 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + prowconfig "k8s.io/test-infra/prow/config" + + jc "github.com/openshift/ci-operator-prowgen/pkg/jobconfig" +) + +type options struct { + prowJobConfigDir string + + help bool +} + +func bindOptions(flag *flag.FlagSet) *options { + opt := &options{} + + flag.StringVar(&opt.prowJobConfigDir, "prow-jobs-dir", "", "Path to a root of directory structure with Prow job config files (ci-operator/jobs in openshift/release)") + flag.BoolVar(&opt.help, "h", false, "Show help for ci-operator-prowgen") + + return opt +} + +func determinizeJobs(prowJobConfigDir string) error { + if err := filepath.Walk(prowJobConfigDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to walk file/directory '%s'", path) + return nil + } + + if !info.IsDir() && filepath.Ext(path) == ".yaml" { + var jobConfig *prowconfig.JobConfig + if jobConfig, err = jc.ReadFromFile(path); err != nil { + fmt.Fprintf(os.Stderr, "Failed to read Prow job config from '%s' (%v)", path, err) + return nil + } + if err := jc.WriteToFile(path, jobConfig); err != nil { + fmt.Fprintf(os.Stderr, "Failed to write Prow job config to '%s' (%v)", path, err) + return nil + } + } + return nil + }); err != nil { + return fmt.Errorf("Failed to determinize all Prow jobs") + } + + return nil +} + +func main() { + flagSet := flag.NewFlagSet("", flag.ExitOnError) + opt := bindOptions(flagSet) + flagSet.Parse(os.Args[1:]) + + if opt.help { + flagSet.Usage() + os.Exit(0) + } + + if len(opt.prowJobConfigDir) > 0 { + if err := determinizeJobs(opt.prowJobConfigDir); err != nil { + fmt.Fprintf(os.Stderr, "determinize failed (%v)\n", err) + + } + } else { + fmt.Fprintf(os.Stderr, "determinize tool needs the --prow-jobs-dir\n") + os.Exit(1) + } +} diff --git a/images/determinize-prow-jobs/Dockerfile b/images/determinize-prow-jobs/Dockerfile new file mode 100644 index 00000000..3b228dae --- /dev/null +++ b/images/determinize-prow-jobs/Dockerfile @@ -0,0 +1,5 @@ +FROM centos:7 +LABEL maintainer="muller@redhat.com" + +ADD determinize-prow-jobs /usr/bin/determinize-prow-jobs +ENTRYPOINT ["/usr/bin/determinize-prow-jobs"] diff --git a/pkg/jobconfig/files.go b/pkg/jobconfig/files.go new file mode 100644 index 00000000..36c377b2 --- /dev/null +++ b/pkg/jobconfig/files.go @@ -0,0 +1,41 @@ +package jobconfig + +import ( + "fmt" + "io/ioutil" + + "github.com/ghodss/yaml" + + prowconfig "k8s.io/test-infra/prow/config" +) + +// ReadFromFile reads Prow job config from a YAML file +func ReadFromFile(path string) (*prowconfig.JobConfig, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read Prow job config (%v)", err) + } + + var jobConfig *prowconfig.JobConfig + if err := yaml.Unmarshal(data, &jobConfig); err != nil { + return nil, fmt.Errorf("failed to load Prow job config (%v)", err) + } + if jobConfig == nil { // happens when `data` is empty + return nil, fmt.Errorf("failed to load Prow job config") + } + + return jobConfig, nil +} + +// WriteToFile writes Prow job config to a YAML file +func WriteToFile(path string, jobConfig *prowconfig.JobConfig) error { + jobConfigAsYaml, err := yaml.Marshal(*jobConfig) + if err != nil { + return fmt.Errorf("failed to marshal the job config (%v)", err) + } + if err := ioutil.WriteFile(path, jobConfigAsYaml, 0664); err != nil { + return fmt.Errorf("Failed to write job config to '%s' (%v)", path, err) + } + + return nil +}