diff --git a/README.md b/README.md index 941933bf..32a4b3ed 100644 --- a/README.md +++ b/README.md @@ -6,48 +6,78 @@ owners need to write when they use component. The generator is able to entirely generate the necessary Prow job configuration from the ci-operator configuration file. +## TL;DR + +**Question:** How do I create Prow jobs running ci-operator for a newly onboarded OpenShift +component? + +**Answer:** +1. Get a working copy of [openshift/release](https://github.com/openshift/release) (we’ll shorten path to it to `$RELEASE`) +2. Create a [ci-operator configuration file](https://github.com/openshift/ci-operator/blob/master/ONBOARD.md#prepare-configuration-for-component-repo) under `$RELEASE/ci-operator/config`, following the `organization/component/branch.json` convention. +3. Run `ci-operator-prowgen --from-dir $RELEASE/ci-operator/config// --to-dir $RELEASE/ci-operator/jobs` +4. Review Prow job configuration files created in `$RELEASE/ci-operator/jobs//` +5. Commit both ci-operator configuration file and Prow job configuration files and issue a PR to upstream. +6. Profit after merge. + ## Use To use the generator, you need to build it: ``` -$ go build ./tools/ci-operator-prowgen +$ make build +``` + +Alternatively, you may obtain a containerized version from the registry on +`api.ci.openshift.org`: + +``` +$ docker pull registry.svc.ci.openshift.org/ci/ci-operator-prowgen:latest ``` -### Full-repository +### Generate Prow jobs for new ci-operator config file -The generator can use the naming convention and directory structure `openshift/release`. -Using this mode, all you need to do is to place your `ci-operator` configuration file -to the correct place in [ci-operator/config](https://github.com/openshift/release/tree/master/ci-operator/config) -directory and then run the generator with `--prow-jobs-dir` and `--config-dir` parameters: +The generator can use the naming conventions and directory structure of the +[openshift/release](https://github.com/openshift/release) repository. Provided +you placed your `ci-operator` configuration file to the correct place in +[ci-operator/config](https://github.com/openshift/release/tree/master/ci-operator/config), +you may run the following (`$REPO is a path to `openshift/release` working +copy): ``` -$ ./ci-operator-prowgen --config-dir ../../ci-operator/config/ --prow-jobs-dir ../../ci-operator/jobs/ +$ ./ci-operator-prowgen --from-file $REPO/ci-operator/config/org/component/branch.json \ + --to-dir $REPO/ci-operator/jobs ``` -This will create Prow job configuration files under the -[ci-operator/jobs](../../ci-operator/jobs) directory, including one for your new -configuration file. The naming structure is the same like in the -`ci-operator/config` directory. +This extracts the `org` and `component` from the configuration file path, reads +the `branch.json` file and generates new Prow job configuration files in the +`(...)/ci-operator/jobs/` directory, creating the necessary directory structure +and files if needed. If the target files already exist and contain Prow job +configuration, newly generated jobs will be merged with the old ones (jobs are +matched by name). + +### Generate Prow jobs for multiple ci-operator config files -### Single configuration +The generator may take a directory as an input. In this case, the generator +walks the directory structure under the given directory, finds all JSON files +there and generates jobs for all of them. -You can use `--source-config` option instead to pass a single `ci-operator` -configuration file. In this case, the generator will print the Prow job config -YAML to the standard output: +You can generate jobs for a certain component, organization, or everything: ``` -$ ./ci-operator-prowgen --source-config path/to/ci-operator/config.json -postsubmits: - openshift/service-serving-cert-signer: - - agent: kubernetes -(...) +$ ./ci-operator-prowgen --from-dir $REPO/ci-operator/config/org/component --to-dir $REPO/ci-operator/jobs +$ ./ci-operator-prowgen --from-dir $REPO/ci-operator/config/org --to-dir $REPO/ci-operator/jobs +$ ./ci-operator-prowgen --from-dir $REPO/ci-operator/config --to-dir $REPO/ci-operator/jobs ``` -Please note that elements of the file path are still used to identify -organization/repo/branch, so the path cannot be entirely arbirary. The path is -expected to have a `(anything)/ORGANIZATION/REPO/BRANCH.extension` form, just -likes path in [ci-operator/config](../..ci-operator/config) do. +If you have cloned `openshift/release` with `go get` and you have `$GOPATH` set +correctly, the generator can derive the paths for the input/output directories. +These invocations are equivalent: + +``` +$ ./ci-operator-prowgen --from-release-repo --to-release-repo +$ ./ci-operator-prowgen --from-dir $GOPATH/src/github.com/openshift/release/ci-operator/config \ + --to-dir $GOPATH/src/github.com/openshift/release/ci-operator/jobs +``` ## What does the generator create @@ -63,7 +93,7 @@ presubmits: - master context: ci/prow/TEST decorate: true - name: pull-ci-ORG-REPO-TEST + name: pull-ci-ORG-REPO-BRANCH-TEST rerun_command: /test TEST skip_cloning: true spec: @@ -97,11 +127,11 @@ way. To build the generator, run: ``` -$ go build ./ci-operator-prowgen +$ make build ``` To run unit-tests, run: ``` -$ go test ./ci-operator-prowgen +$ make test ``` diff --git a/cmd/ci-operator-prowgen/main.go b/cmd/ci-operator-prowgen/main.go index 68fcaabd..1e48d236 100644 --- a/cmd/ci-operator-prowgen/main.go +++ b/cmd/ci-operator-prowgen/main.go @@ -19,33 +19,59 @@ import ( ) type options struct { - ciOperatorConfigPath string - prowJobConfigPath string + fromFile string + fromDir string + fromReleaseRepo bool - fullRepoMode bool + toFile string + toDir string + toReleaseRepo bool - ciOperatorConfigDir string - prowJobConfigDir string - - help bool - verbose bool + help bool } func bindOptions(flag *flag.FlagSet) *options { opt := &options{} - flag.StringVar(&opt.ciOperatorConfigPath, "source-config", "", "Path to ci-operator configuration file in openshift/release repository.") - flag.StringVar(&opt.prowJobConfigPath, "target-job-config", "", "Path to a file wher Prow job config will be written. If the file already exists and contains Prow job config, generated jobs will be merged with existing ones") + flag.StringVar(&opt.fromFile, "from-file", "", "Path to a ci-operator configuration file") + flag.StringVar(&opt.fromDir, "from-dir", "", "Path to a directory with a directory structure holding ci-operator configuration files for multiple components") + flag.BoolVar(&opt.fromReleaseRepo, "from-release-repo", false, "If set, it behaves like --from-dir=$GOPATH/src/github.com/openshift/release/ci-operator/config") - flag.StringVar(&opt.ciOperatorConfigDir, "config-dir", "", "Path to a root of directory structure with ci-operator config files (ci-operator/config in openshift/release)") - flag.StringVar(&opt.prowJobConfigDir, "prow-jobs-dir", "", "Path to a root of directory structure with Prow job config files (ci-operator/jobs in openshift/release)") + flag.StringVar(&opt.toFile, "to-file", "", "Path to a Prow job configuration file where new jobs will be added. If the file does not exist, it will be created") + flag.StringVar(&opt.toDir, "to-dir", "", "Path to a directory with a directory structure holding Prow job configuration files for multiple components") + flag.BoolVar(&opt.toReleaseRepo, "to-release-repo", false, "If set, it behaves like --to-dir=$GOPATH/src/github.com/openshift/release/ci-operator/jobs") flag.BoolVar(&opt.help, "h", false, "Show help for ci-operator-prowgen") - flag.BoolVar(&opt.verbose, "v", false, "Show verbose output") return opt } +func (o *options) process() error { + var err error + + if o.fromReleaseRepo { + if o.fromDir, err = getReleaseRepoDir("ci-operator/config"); err != nil { + return fmt.Errorf("--from-release-repo error: %v", err) + } + } + + if o.toReleaseRepo { + if o.toDir, err = getReleaseRepoDir("ci-operator/jobs"); err != nil { + return fmt.Errorf("--to-release-repo error: %v", err) + } + } + + if (o.fromFile == "" && o.fromDir == "") || (o.fromFile != "" && o.fromDir != "") { + return fmt.Errorf("ci-operator-prowgen needs exactly one of `--from-{file,dir,release-repo}` options") + } + + if (o.toFile == "" && o.toDir == "") || (o.toFile != "" && o.toDir != "") { + return fmt.Errorf("ci-operator-prowgen needs exactly one of `--to-{file,dir,release-repo}` options") + } + + return nil +} + // Generate a PodSpec that runs `ci-operator`, to be used in Presubmit/Postsubmit // Various pieces are derived from `org`, `repo`, `branch` and `target`. // `additionalArgs` are passed as additional arguments to `ci-operator` @@ -246,7 +272,7 @@ func writeJobsIntoComponentDirectory(jobDir, org, repo string, jobConfig *prowco // Iterate over all ci-operator config files under a given path and generate a // Prow job configuration files for each one under a different path, mimicking // the directory structure. -func generateAllProwJobs(configDir, jobDir string) error { +func generateJobsFromDirectory(configDir, jobDir, jobFile string) error { err := filepath.Walk(configDir, func(path string, info os.FileInfo, err error) error { if err != nil { fmt.Fprintf(os.Stderr, "Error encontered while generating Prow job config: %v\n", err) @@ -258,15 +284,21 @@ func generateAllProwJobs(configDir, jobDir string) error { return err } - if err = writeJobsIntoComponentDirectory(jobDir, org, repo, jobConfig); err != nil { - return err + if len(jobDir) > 0 { + if err = writeJobsIntoComponentDirectory(jobDir, org, repo, jobConfig); err != nil { + return err + } + } else if len(jobFile) > 0 { + if err = mergeJobsIntoFile(jobFile, jobConfig); err != nil { + return err + } } } return nil }) if err != nil { - return fmt.Errorf("Failed to generate all Prow jobs") + return fmt.Errorf("failed to generate all Prow jobs (%v)", err) } return nil @@ -383,6 +415,18 @@ func mergeJobsIntoFile(prowConfigPath string, jobConfig *prowconfig.JobConfig) e return nil } +func getReleaseRepoDir(directory string) (string, error) { + var gopath string + if gopath = os.Getenv("GOPATH"); len(gopath) == 0 { + return "", fmt.Errorf("GOPATH not set, cannot infer openshift/release repo location") + } + tentative := filepath.Join(gopath, "src/github.com/openshift/release", directory) + if stat, err := os.Stat(tentative); err == nil && stat.IsDir() { + return tentative, nil + } + return "", fmt.Errorf("%s is not an existing directory", tentative) +} + func main() { flagSet := flag.NewFlagSet("", flag.ExitOnError) opt := bindOptions(flagSet) @@ -393,26 +437,32 @@ func main() { os.Exit(0) } - if len(opt.ciOperatorConfigPath) > 0 { - if jobConfig, _, _, err := generateProwJobsFromConfigFile(opt.ciOperatorConfigPath); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + if err := opt.process(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + + if len(opt.fromFile) > 0 { + jobConfig, org, repo, err := generateProwJobsFromConfigFile(opt.fromFile) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to generate jobs from '%s' (%v)\n", opt.fromFile, err) os.Exit(1) - } else { - if len(opt.prowJobConfigPath) > 0 { - err = mergeJobsIntoFile(opt.prowJobConfigPath, jobConfig) - } else { - err = writeJobs(jobConfig) + } + if len(opt.toFile) > 0 { // from file to file + if err := mergeJobsIntoFile(opt.toFile, jobConfig); err != nil { + fmt.Fprintf(os.Stderr, "failed to write jobs to '%s' (%v)\n", opt.toFile, err) + os.Exit(1) } - if err != nil { - fmt.Fprintf(os.Stderr, "failed to write the job configuration (%v)\n", err) + } else { // from file to directory + if err := writeJobsIntoComponentDirectory(opt.toDir, org, repo, jobConfig); err != nil { + fmt.Fprintf(os.Stderr, "failed to write jobs to '%s' (%v)\n", opt.toDir, err) os.Exit(1) } - } - } else if len(opt.ciOperatorConfigDir) > 0 && len(opt.prowJobConfigDir) > 0 { - generateAllProwJobs(opt.ciOperatorConfigDir, opt.prowJobConfigDir) - } else { - fmt.Fprintf(os.Stderr, "ci-operator-prowgen needs --source-config, or --{config,prow-jobs}-dir option\n") - os.Exit(1) + } else { // from directory + if err := generateJobsFromDirectory(opt.fromDir, opt.toDir, opt.toFile); err != nil { + fmt.Fprintf(os.Stderr, "failed to generate jobs from '%s' (%v)\n", opt.fromDir, err) + os.Exit(1) + } } }