Improve the frontend

README.md

@@ -6,48 +6,78 @@ owners need to write when they use
 component. The generator is able to entirely generate the necessary Prow job
 configuration from the ci-operator configuration file.
 
+## TL;DR
+
+**Question:** How do I create Prow jobs running ci-operator for a newly onboarded OpenShift
+component?
+
+**Answer:**
+1. Get a working copy of [openshift/release](https://github.com/openshift/release) (we’ll shorten path to it to `$RELEASE`)
+2. Create a [ci-operator configuration file](https://github.com/openshift/ci-operator/blob/master/ONBOARD.md#prepare-configuration-for-component-repo) under `$RELEASE/ci-operator/config`, following the `organization/component/branch.json` convention.
+3. Run `ci-operator-prowgen --from-dir $RELEASE/ci-operator/config/<org>/<component> --to-dir $RELEASE/ci-operator/jobs`
+4. Review Prow job configuration files created in `$RELEASE/ci-operator/jobs/<org>/<component>` 
+5. Commit both ci-operator configuration file and Prow job configuration files and issue a PR to upstream.
+6. Profit after merge.
+
 ## Use
 
 To use the generator, you need to build it:
 
 ```
-$ go build ./tools/ci-operator-prowgen
+$ make build
+```
+
+Alternatively, you may obtain a containerized version from the registry on
+`api.ci.openshift.org`:
+
+```
+$ docker pull registry.svc.ci.openshift.org/ci/ci-operator-prowgen:latest
 ```
 
-### Full-repository
+### Generate Prow jobs for new ci-operator config file
 
-The generator can use the naming convention and directory structure `openshift/release`.
-Using this mode, all you need to do is to place your `ci-operator` configuration file
-to the correct place in [ci-operator/config](https://github.com/openshift/release/tree/master/ci-operator/config)
-directory and then run the generator with `--prow-jobs-dir` and `--config-dir` parameters:
+The generator can use the naming conventions and directory structure of the
+[openshift/release](https://github.com/openshift/release) repository. Provided
+you placed your `ci-operator` configuration file to the correct place in
+[ci-operator/config](https://github.com/openshift/release/tree/master/ci-operator/config),
+you may run the following (`$REPO is a path to `openshift/release` working
+copy):
 
 ```
-$ ./ci-operator-prowgen --config-dir ../../ci-operator/config/ --prow-jobs-dir ../../ci-operator/jobs/
+$ ./ci-operator-prowgen --from-file $REPO/ci-operator/config/org/component/branch.json \
+ --to-dir $REPO/ci-operator/jobs
 ```
 
-This will create Prow job configuration files under the
-[ci-operator/jobs](../../ci-operator/jobs) directory, including one for your new
-configuration file. The naming structure is the same like in the
-`ci-operator/config` directory.
+This extracts the `org` and `component` from the configuration file path, reads
+the `branch.json` file and generates new Prow job configuration files in the
+`(...)/ci-operator/jobs/` directory, creating the necessary directory structure
+and files if needed. If the target files already exist and contain Prow job
+configuration, newly generated jobs will be merged with the old ones (jobs are
+matched by name).
+
+### Generate Prow jobs for multiple ci-operator config files
 
-### Single configuration
+The generator may take a directory as an input. In this case, the generator
+walks the directory structure under the given directory, finds all JSON files
+there and generates jobs for all of them.
 
-You can use `--source-config` option instead to pass a single `ci-operator`
-configuration file. In this case, the generator will print the Prow job config
-YAML to the standard output:
+You can generate jobs for a certain component, organization, or everything:
 
 ```
-$ ./ci-operator-prowgen --source-config path/to/ci-operator/config.json
-postsubmits:
-  openshift/service-serving-cert-signer:
-  - agent: kubernetes
-(...)
+$ ./ci-operator-prowgen --from-dir $REPO/ci-operator/config/org/component --to-dir $REPO/ci-operator/jobs
+$ ./ci-operator-prowgen --from-dir $REPO/ci-operator/config/org --to-dir $REPO/ci-operator/jobs
+$ ./ci-operator-prowgen --from-dir $REPO/ci-operator/config --to-dir $REPO/ci-operator/jobs
 ```
 
-Please note that elements of the file path are still used to identify
-organization/repo/branch, so the path cannot be entirely arbirary. The path is
-expected to have a `(anything)/ORGANIZATION/REPO/BRANCH.extension` form, just
-likes path in [ci-operator/config](../..ci-operator/config) do.
+If you have cloned `openshift/release` with `go get` and you have `$GOPATH` set
+correctly, the generator can derive the paths for the input/output directories.
+These invocations are equivalent:
+
+```
+$ ./ci-operator-prowgen --from-release-repo --to-release-repo
+$ ./ci-operator-prowgen --from-dir $GOPATH/src/github.com/openshift/release/ci-operator/config \
+ --to-dir $GOPATH/src/github.com/openshift/release/ci-operator/jobs
+```
 
 ## What does the generator create
 
@@ -63,7 +93,7 @@ presubmits:
     - master
     context: ci/prow/TEST
     decorate: true
-    name: pull-ci-ORG-REPO-TEST
+    name: pull-ci-ORG-REPO-BRANCH-TEST
     rerun_command: /test TEST
     skip_cloning: true
     spec:
@@ -97,11 +127,11 @@ way.
 To build the generator, run:
 
 ```
-$ go build ./ci-operator-prowgen
+$ make build
 ```
 
 To run unit-tests, run:
 
 ```
-$ go test ./ci-operator-prowgen
+$ make test
 ```

cmd/ci-operator-prowgen/main.go

@@ -19,29 +19,29 @@ import (
 )
 
 type options struct {
-	ciOperatorConfigPath string
-	prowJobConfigPath    string
+	fromFile        string
+	fromDir         string
+	fromReleaseRepo bool
 
-	fullRepoMode bool
+	toFile        string
+	toDir         string
+	toReleaseRepo bool
 
-	ciOperatorConfigDir string
-	prowJobConfigDir    string
-
-	help    bool
-	verbose bool
+	help bool
 }
 
 func bindOptions(flag *flag.FlagSet) *options {
 	opt := &options{}
 
-	flag.StringVar(&opt.ciOperatorConfigPath, "source-config", "", "Path to ci-operator configuration file in openshift/release repository.")
-	flag.StringVar(&opt.prowJobConfigPath, "target-job-config", "", "Path to a file wher Prow job config will be written. If the file already exists and contains Prow job config, generated jobs will be merged with existing ones")
+	flag.StringVar(&opt.fromFile, "from-file", "", "Path to a ci-operator configuration file")
+	flag.StringVar(&opt.fromDir, "from-dir", "", "Path to a directory with a directory structure holding ci-operator configuration files for multiple components")
+	flag.BoolVar(&opt.fromReleaseRepo, "from-release-repo", false, "If set, it behaves like --from-dir=$GOPATH/src/github.com/openshift/release/ci-operator/config")
 
-	flag.StringVar(&opt.ciOperatorConfigDir, "config-dir", "", "Path to a root of directory structure with ci-operator config files (ci-operator/config in openshift/release)")
-	flag.StringVar(&opt.prowJobConfigDir, "prow-jobs-dir", "", "Path to a root of directory structure with Prow job config files (ci-operator/jobs in openshift/release)")
+	flag.StringVar(&opt.toFile, "to-file", "", "Path to a Prow job configuration file where new jobs will be added. If the file does not exist, it will be created")
+	flag.StringVar(&opt.toDir, "to-dir", "", "Path to a directory with a directory structure holding Prow job configuration files for multiple components")
+	flag.BoolVar(&opt.toReleaseRepo, "to-release-repo", false, "If set, it behaves like --to-dir=$GOPATH/src/github.com/openshift/release/ci-operator/jobs")
 
 	flag.BoolVar(&opt.help, "h", false, "Show help for ci-operator-prowgen")
-	flag.BoolVar(&opt.verbose, "v", false, "Show verbose output")
 
 	return opt
 }
@@ -246,7 +246,7 @@ func writeJobsIntoComponentDirectory(jobDir, org, repo string, jobConfig *prowco
 // Iterate over all ci-operator config files under a given path and generate a
 // Prow job configuration files for each one under a different path, mimicking
 // the directory structure.
-func generateAllProwJobs(configDir, jobDir string) error {
+func generateJobsFromDirectory(configDir, jobDir, jobFile string) error {
 	err := filepath.Walk(configDir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error encontered while generating Prow job config: %v\n", err)
@@ -258,15 +258,21 @@ func generateAllProwJobs(configDir, jobDir string) error {
 				return err
 			}
 
-			if err = writeJobsIntoComponentDirectory(jobDir, org, repo, jobConfig); err != nil {
-				return err
+			if len(jobDir) > 0 {
+				if err = writeJobsIntoComponentDirectory(jobDir, org, repo, jobConfig); err != nil {
+					return err
+				}
+			} else if len(jobFile) > 0 {
+				if err = mergeJobsIntoFile(jobFile, jobConfig); err != nil {
+					return err
+				}
 			}
 		}
 		return nil
 	})
 
 	if err != nil {
-		return fmt.Errorf("Failed to generate all Prow jobs")
+		return fmt.Errorf("failed to generate all Prow jobs (%v)", err)
 	}
 
 	return nil
@@ -383,6 +389,18 @@ func mergeJobsIntoFile(prowConfigPath string, jobConfig *prowconfig.JobConfig) e
 	return nil
 }
 
+func getReleaseRepoDir(directory string) (string, error) {
+	var gopath string
+	if gopath = os.Getenv("GOPATH"); len(gopath) == 0 {
+		return "", fmt.Errorf("GOPATH not set, cannot infer openshift/release repo location")
+	}
+	tentative := filepath.Join(gopath, "src/github.com/openshift/release", directory)
+	if stat, err := os.Stat(tentative); err == nil && stat.IsDir() {
+		return tentative, nil
+	}
+	return "", fmt.Errorf("%s is not an existing directory", tentative)
+}
+
 func main() {
 	flagSet := flag.NewFlagSet("", flag.ExitOnError)
 	opt := bindOptions(flagSet)
@@ -393,26 +411,54 @@ func main() {
 		os.Exit(0)
 	}
 
-	if len(opt.ciOperatorConfigPath) > 0 {
-		if jobConfig, _, _, err := generateProwJobsFromConfigFile(opt.ciOperatorConfigPath); err != nil {
-			fmt.Fprintf(os.Stderr, "%v\n", err)
+	if opt.fromReleaseRepo {
+		var err error
+		if opt.fromDir, err = getReleaseRepoDir("ci-operator/config"); err != nil {
+			fmt.Fprintf(os.Stderr, "--from-release-repo error: %v\n", err)
 			os.Exit(1)
-		} else {
-			if len(opt.prowJobConfigPath) > 0 {
-				err = mergeJobsIntoFile(opt.prowJobConfigPath, jobConfig)
-			} else {
-				err = writeJobs(jobConfig)
+		}
+	}
+	if opt.toReleaseRepo {
+		var err error
+		if opt.toDir, err = getReleaseRepoDir("ci-operator/jobs"); err != nil {
+			fmt.Fprintf(os.Stderr, "--to-release-repo error: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
+	if len(opt.fromFile) > 0 {
+
+		jobConfig, org, repo, err := generateProwJobsFromConfigFile(opt.fromFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to generate jobs from '%s' (%v)\n", opt.fromFile, err)
+			os.Exit(1)
+		}
+		if len(opt.toFile) > 0 {
+			if err := mergeJobsIntoFile(opt.toFile, jobConfig); err != nil {
+				fmt.Fprintf(os.Stderr, "failed to write jobs to '%s' (%v)\n", opt.toFile, err)
+				os.Exit(1)
 			}
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "failed to write the job configuration (%v)\n", err)
+		} else if len(opt.toDir) > 0 {
+			if err := writeJobsIntoComponentDirectory(opt.toDir, org, repo, jobConfig); err != nil {
+				fmt.Fprintf(os.Stderr, "failed to write jobs to '%s' (%v)\n", opt.toDir, err)
 				os.Exit(1)
 			}
-
+		} else {
+			fmt.Fprintf(os.Stderr, "ci-operator-prowgen needs at least one of `--to-{file,dir,release-repo}` options\n")
+			os.Exit(1)
+		}
+	} else if len(opt.fromDir) > 0 {
+		if len(opt.toFile) > 0 || len(opt.toDir) > 0 {
+			if err := generateJobsFromDirectory(opt.fromDir, opt.toDir, opt.toFile); err != nil {
+				fmt.Fprintf(os.Stderr, "failed to generate jobs from '%s' (%v)\n", opt.fromDir, err)
+				os.Exit(1)
+			}
+		} else {
+			fmt.Fprintf(os.Stderr, "ci-operator-prowgen needs at least one of `--to-{file,dir,release-repo}` options\n")
+			os.Exit(1)
 		}
-	} else if len(opt.ciOperatorConfigDir) > 0 && len(opt.prowJobConfigDir) > 0 {
-		generateAllProwJobs(opt.ciOperatorConfigDir, opt.prowJobConfigDir)
 	} else {
-		fmt.Fprintf(os.Stderr, "ci-operator-prowgen needs --source-config, or --{config,prow-jobs}-dir option\n")
+		fmt.Fprintf(os.Stderr, "ci-operator-prowgen needs at least one of `--from-{file,dir,release-repo}` options\n")
 		os.Exit(1)
 	}
 }