Add a simple program to make Prow job config ordering deterministic

cmd/ci-operator-prowgen/main.go

@@ -10,12 +10,12 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/ghodss/yaml"
-
 	cioperatorapi "github.com/openshift/ci-operator/pkg/api"
 	kubeapi "k8s.io/api/core/v1"
 	prowconfig "k8s.io/test-infra/prow/config"
 	prowkube "k8s.io/test-infra/prow/kube"
+
+	jc "github.com/openshift/ci-operator-prowgen/pkg/jobconfig"
 )
 
 type options struct {
@@ -304,46 +304,6 @@ func generateJobsFromDirectory(configDir, jobDir, jobFile string) error {
 	return nil
 }
 
-func readJobConfig(path string) (*prowconfig.JobConfig, error) {
-	data, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read Prow job config (%v)", err)
-	}
-
-	var jobConfig *prowconfig.JobConfig
-	if err := yaml.Unmarshal(data, &jobConfig); err != nil {
-		return nil, fmt.Errorf("failed to load Prow job config (%v)", err)
-	}
-	if jobConfig == nil { // happens when `data` is empty
-		return nil, fmt.Errorf("failed to load Prow job config")
-	}
-
-	return jobConfig, nil
-}
-
-// Print JobConfig to stdout as YAML
-func writeJobs(jobConfig *prowconfig.JobConfig) error {
-	jobConfigAsYaml, err := yaml.Marshal(*jobConfig)
-	if err != nil {
-		return fmt.Errorf("failed to marshal the job config (%v)", err)
-	}
-	fmt.Printf(string(jobConfigAsYaml))
-	return nil
-}
-
-// Write JobConfig to a file as YAML
-func writeJobsToFile(path string, jobConfig *prowconfig.JobConfig) error {
-	jobConfigAsYaml, err := yaml.Marshal(*jobConfig)
-	if err != nil {
-		return fmt.Errorf("failed to marshal the job config (%v)", err)
-	}
-	if err := ioutil.WriteFile(path, jobConfigAsYaml, 0664); err != nil {
-		return fmt.Errorf("Failed to write job config to '%s' (%v)", path, err)
-	}
-
-	return nil
-}
-
 // Given two JobConfig, merge jobs from the `source` one to to `destination`
 // one. Jobs are matched by name. All jobs from `source` will be present in
 // `destination` - if there were jobs with the same name in `destination`, they
@@ -401,14 +361,14 @@ func mergeJobConfig(destination, source *prowconfig.JobConfig) {
 // to the file path. If the file already contains some jobs, new ones will be
 // merged with the existing ones.
 func mergeJobsIntoFile(prowConfigPath string, jobConfig *prowconfig.JobConfig) error {
-	existingJobConfig, err := readJobConfig(prowConfigPath)
+	existingJobConfig, err := jc.ReadFromFile(prowConfigPath)
 	if err != nil {
 		existingJobConfig = &prowconfig.JobConfig{}
 	}
 
 	mergeJobConfig(existingJobConfig, jobConfig)
 
-	if err = writeJobsToFile(prowConfigPath, existingJobConfig); err != nil {
+	if err = jc.WriteToFile(prowConfigPath, existingJobConfig); err != nil {
 		return err
 	}
 

cmd/determinize-prow-jobs/main.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	prowconfig "k8s.io/test-infra/prow/config"
+
+	jc "github.com/openshift/ci-operator-prowgen/pkg/jobconfig"
+)
+
+type options struct {
+	prowJobConfigDir string
+
+	help bool
+}
+
+func bindOptions(flag *flag.FlagSet) *options {
+	opt := &options{}
+
+	flag.StringVar(&opt.prowJobConfigDir, "prow-jobs-dir", "", "Path to a root of directory structure with Prow job config files (ci-operator/jobs in openshift/release)")
+	flag.BoolVar(&opt.help, "h", false, "Show help for ci-operator-prowgen")
+
+	return opt
+}
+
+func determinizeJobs(prowJobConfigDir string) error {
+	if err := filepath.Walk(prowJobConfigDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to walk file/directory '%s'", path)
+			return nil
+		}
+
+		if !info.IsDir() && filepath.Ext(path) == ".yaml" {
+			var jobConfig *prowconfig.JobConfig
+			if jobConfig, err = jc.ReadFromFile(path); err != nil {
+				fmt.Fprintf(os.Stderr, "Failed to read Prow job config from '%s' (%v)", path, err)
+				return nil
+			}
+			if err := jc.WriteToFile(path, jobConfig); err != nil {
+				fmt.Fprintf(os.Stderr, "Failed to write Prow job config to '%s' (%v)", path, err)
+				return nil
+			}
+		}
+		return nil
+	}); err != nil {
+		return fmt.Errorf("Failed to determinize all Prow jobs")
+	}
+
+	return nil
+}
+
+func main() {
+	flagSet := flag.NewFlagSet("", flag.ExitOnError)
+	opt := bindOptions(flagSet)
+	flagSet.Parse(os.Args[1:])
+
+	if opt.help {
+		flagSet.Usage()
+		os.Exit(0)
+	}
+
+	if len(opt.prowJobConfigDir) > 0 {
+		if err := determinizeJobs(opt.prowJobConfigDir); err != nil {
+			fmt.Fprintf(os.Stderr, "determinize failed (%v)\n", err)
+
+		}
+	} else {
+		fmt.Fprintf(os.Stderr, "determinize tool needs the --prow-jobs-dir\n")
+		os.Exit(1)
+	}
+}

images/determinize-prow-jobs/Dockerfile

@@ -0,0 +1,5 @@
+FROM centos:7
+LABEL maintainer="[email protected]"
+
+ADD determinize-prow-jobs /usr/bin/determinize-prow-jobs
+ENTRYPOINT ["/usr/bin/determinize-prow-jobs"]

pkg/jobconfig/files.go

@@ -0,0 +1,41 @@
+package jobconfig
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/ghodss/yaml"
+
+	prowconfig "k8s.io/test-infra/prow/config"
+)
+
+// ReadFromFile reads Prow job config from a YAML file
+func ReadFromFile(path string) (*prowconfig.JobConfig, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read Prow job config (%v)", err)
+	}
+
+	var jobConfig *prowconfig.JobConfig
+	if err := yaml.Unmarshal(data, &jobConfig); err != nil {
+		return nil, fmt.Errorf("failed to load Prow job config (%v)", err)
+	}
+	if jobConfig == nil { // happens when `data` is empty
+		return nil, fmt.Errorf("failed to load Prow job config")
+	}
+
+	return jobConfig, nil
+}
+
+// WriteToFile writes Prow job config to a YAML file
+func WriteToFile(path string, jobConfig *prowconfig.JobConfig) error {
+	jobConfigAsYaml, err := yaml.Marshal(*jobConfig)
+	if err != nil {
+		return fmt.Errorf("failed to marshal the job config (%v)", err)
+	}
+	if err := ioutil.WriteFile(path, jobConfigAsYaml, 0664); err != nil {
+		return fmt.Errorf("Failed to write job config to '%s' (%v)", path, err)
+	}
+
+	return nil
+}