diff --git a/README.md b/README.md index 116e71bc..021ebec9 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ component? **Answer:** 1. Get a working copy of [openshift/release](https://github.com/openshift/release) (we’ll shorten path to it to `$RELEASE`) -2. Create a [ci-operator configuration file](https://github.com/openshift/ci-operator/blob/master/ONBOARD.md#prepare-configuration-for-component-repo) under `$RELEASE/ci-operator/config`, following the `organization/component/branch.json` convention. +2. Create a [ci-operator configuration file](https://github.com/openshift/ci-operator/blob/master/ONBOARD.md#prepare-configuration-for-component-repo) under `$RELEASE/ci-operator/config`, following the `organization/component/branch.yaml` convention. 3. Run `ci-operator-prowgen --from-dir $RELEASE/ci-operator/config// --to-dir $RELEASE/ci-operator/jobs` 4. Review Prow job configuration files created in `$RELEASE/ci-operator/jobs//` 5. Commit both ci-operator configuration file and Prow job configuration files and issue a PR to upstream. @@ -44,12 +44,12 @@ you may run the following (`$REPO is a path to `openshift/release` working copy): ``` -$ ./ci-operator-prowgen --from-file $REPO/ci-operator/config/org/component/branch.json \ +$ ./ci-operator-prowgen --from-file $REPO/ci-operator/config/org/component/branch.yaml \ --to-dir $REPO/ci-operator/jobs ``` This extracts the `org` and `component` from the configuration file path, reads -the `branch.json` file and generates new Prow job configuration files in the +the `branch.yaml` file and generates new Prow job configuration files in the `(...)/ci-operator/jobs/` directory, creating the necessary directory structure and files if needed. If the target files already exist and contain Prow job configuration, newly generated jobs will be merged with the old ones (jobs are @@ -58,7 +58,7 @@ matched by name). ### Generate Prow jobs for multiple ci-operator config files The generator may take a directory as an input. In this case, the generator -walks the directory structure under the given directory, finds all JSON files +walks the directory structure under the given directory, finds all YAML files there and generates jobs for all of them. You can generate jobs for a certain component, organization, or everything: diff --git a/cmd/ci-operator-prowgen/main.go b/cmd/ci-operator-prowgen/main.go index 0b832fd4..d33146d2 100644 --- a/cmd/ci-operator-prowgen/main.go +++ b/cmd/ci-operator-prowgen/main.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "flag" "fmt" "io/ioutil" @@ -9,6 +8,8 @@ import ( "path/filepath" "strings" + "github.com/ghodss/yaml" + cioperatorapi "github.com/openshift/ci-operator/pkg/api" kubeapi "k8s.io/api/core/v1" prowconfig "k8s.io/test-infra/prow/config" @@ -74,13 +75,13 @@ func (o *options) process() error { // Generate a PodSpec that runs `ci-operator`, to be used in Presubmit/Postsubmit // Various pieces are derived from `org`, `repo`, `branch` and `target`. // `additionalArgs` are passed as additional arguments to `ci-operator` -func generatePodSpec(org, repo, branch, target string, additionalArgs ...string) *kubeapi.PodSpec { +func generatePodSpec(org, repo, configFile, target string, additionalArgs ...string) *kubeapi.PodSpec { configMapKeyRef := kubeapi.EnvVarSource{ ConfigMapKeyRef: &kubeapi.ConfigMapKeySelector{ LocalObjectReference: kubeapi.LocalObjectReference{ Name: fmt.Sprintf("ci-operator-%s-%s", org, repo), }, - Key: fmt.Sprintf("%s.json", branch), + Key: configFile, }, } @@ -91,12 +92,7 @@ func generatePodSpec(org, repo, branch, target string, additionalArgs ...string) Image: "ci-operator:latest", Command: []string{"ci-operator"}, Args: append([]string{"--artifact-dir=$(ARTIFACTS)", fmt.Sprintf("--target=%s", target)}, additionalArgs...), - Env: []kubeapi.EnvVar{ - { - Name: "CONFIG_SPEC", - ValueFrom: &configMapKeyRef, - }, - }, + Env: []kubeapi.EnvVar{{Name: "CONFIG_SPEC", ValueFrom: &configMapKeyRef}}, }, }, } @@ -108,15 +104,15 @@ type testDescription struct { } // Generate a Presubmit job for the given parameters -func generatePresubmitForTest(test testDescription, org, repo, branch string) *prowconfig.Presubmit { +func generatePresubmitForTest(test testDescription, repoInfo *configFilePathElements) *prowconfig.Presubmit { return &prowconfig.Presubmit{ Agent: "kubernetes", AlwaysRun: true, - Brancher: prowconfig.Brancher{Branches: []string{branch}}, + Brancher: prowconfig.Brancher{Branches: []string{repoInfo.branch}}, Context: fmt.Sprintf("ci/prow/%s", test.Name), - Name: fmt.Sprintf("pull-ci-%s-%s-%s-%s", org, repo, branch, test.Name), + Name: fmt.Sprintf("pull-ci-%s-%s-%s-%s", repoInfo.org, repoInfo.repo, repoInfo.branch, test.Name), RerunCommand: fmt.Sprintf("/test %s", test.Name), - Spec: generatePodSpec(org, repo, branch, test.Target), + Spec: generatePodSpec(repoInfo.org, repoInfo.repo, repoInfo.configFilename, test.Target), Trigger: fmt.Sprintf(`((?m)^/test( all| %s),?(\\s+|$))`, test.Name), UtilityConfig: prowconfig.UtilityConfig{ DecorationConfig: &prowkube.DecorationConfig{SkipCloning: true}, @@ -126,12 +122,16 @@ func generatePresubmitForTest(test testDescription, org, repo, branch string) *p } // Generate a Presubmit job for the given parameters -func generatePostsubmitForTest(test testDescription, org, repo, branch string, labels map[string]string, additionalArgs ...string) *prowconfig.Postsubmit { +func generatePostsubmitForTest( + test testDescription, + repoInfo *configFilePathElements, + labels map[string]string, + additionalArgs ...string) *prowconfig.Postsubmit { return &prowconfig.Postsubmit{ Agent: "kubernetes", - Brancher: prowconfig.Brancher{Branches: []string{branch}}, - Name: fmt.Sprintf("branch-ci-%s-%s-%s-%s", org, repo, branch, test.Name), - Spec: generatePodSpec(org, repo, branch, test.Target, additionalArgs...), + Brancher: prowconfig.Brancher{Branches: []string{repoInfo.branch}}, + Name: fmt.Sprintf("branch-ci-%s-%s-%s-%s", repoInfo.org, repoInfo.repo, repoInfo.branch, test.Name), + Spec: generatePodSpec(repoInfo.org, repoInfo.repo, repoInfo.configFilename, test.Target, additionalArgs...), Labels: labels, UtilityConfig: prowconfig.UtilityConfig{ DecorationConfig: &prowkube.DecorationConfig{SkipCloning: true}, @@ -161,23 +161,22 @@ func extractPromotionNamespace(configSpec *cioperatorapi.ReleaseBuildConfigurati // presubmit and postsubmit that has `--target=[images]`. This postsubmit // will additionally pass `--promote` to ci-operator func generateJobs( - configSpec *cioperatorapi.ReleaseBuildConfiguration, - org, repo, branch string, + configSpec *cioperatorapi.ReleaseBuildConfiguration, repoInfo *configFilePathElements, ) *prowconfig.JobConfig { - orgrepo := fmt.Sprintf("%s/%s", org, repo) + orgrepo := fmt.Sprintf("%s/%s", repoInfo.org, repoInfo.repo) presubmits := map[string][]prowconfig.Presubmit{} postsubmits := map[string][]prowconfig.Postsubmit{} for _, element := range configSpec.Tests { test := testDescription{Name: element.As, Target: element.As} - presubmits[orgrepo] = append(presubmits[orgrepo], *generatePresubmitForTest(test, org, repo, branch)) + presubmits[orgrepo] = append(presubmits[orgrepo], *generatePresubmitForTest(test, repoInfo)) } if len(configSpec.Images) > 0 { test := testDescription{Name: "images", Target: "[images]"} - presubmits[orgrepo] = append(presubmits[orgrepo], *generatePresubmitForTest(test, org, repo, branch)) + presubmits[orgrepo] = append(presubmits[orgrepo], *generatePresubmitForTest(test, repoInfo)) // If the images are promoted to 'openshift' namespace, we may want to add // 'artifacts: images' label to the [images] postsubmit. @@ -185,7 +184,7 @@ func generateJobs( if extractPromotionNamespace(configSpec) == "openshift" { labels["artifacts"] = "images" } - imagesPostsubmit := generatePostsubmitForTest(test, org, repo, branch, labels, "--promote") + imagesPostsubmit := generatePostsubmitForTest(test, repoInfo, labels, "--promote") postsubmits[orgrepo] = append(postsubmits[orgrepo], *imagesPostsubmit) } @@ -202,48 +201,57 @@ func readCiOperatorConfig(configFilePath string) (*cioperatorapi.ReleaseBuildCon } var configSpec *cioperatorapi.ReleaseBuildConfiguration - if err := json.Unmarshal(data, &configSpec); err != nil { + if err := yaml.Unmarshal(data, &configSpec); err != nil { return nil, fmt.Errorf("failed to load ci-operator config (%v)", err) } return configSpec, nil } +// path to ci-operator configuration file encodes information about tested code +// .../$ORGANIZATION/$REPOSITORY/$BRANCH.$EXT +type configFilePathElements struct { + org string + repo string + branch string + configFilename string +} + // We use the directory/file naming convention to encode useful information // about component repository information. // The convention for ci-operator config files in this repo: -// ci-operator/config/ORGANIZATION/COMPONENT/BRANCH.json -func extractRepoElementsFromPath(configFilePath string) (string, string, string, error) { +// ci-operator/config/ORGANIZATION/COMPONENT/BRANCH.yaml +func extractRepoElementsFromPath(configFilePath string) (*configFilePathElements, error) { configSpecDir := filepath.Dir(configFilePath) repo := filepath.Base(configSpecDir) if repo == "." || repo == "/" { - return "", "", "", fmt.Errorf("Could not extract repo from '%s' (expected path like '.../ORG/REPO/BRANCH.json", configFilePath) + return nil, fmt.Errorf("Could not extract repo from '%s' (expected path like '.../ORG/REPO/BRANCH.yaml", configFilePath) } org := filepath.Base(filepath.Dir(configSpecDir)) if org == "." || org == "/" { - return "", "", "", fmt.Errorf("Could not extract org from '%s' (expected path like '.../ORG/REPO/BRANCH.json", configFilePath) + return nil, fmt.Errorf("Could not extract org from '%s' (expected path like '.../ORG/REPO/BRANCH.yaml", configFilePath) } - branch := strings.TrimSuffix(filepath.Base(configFilePath), filepath.Ext(configFilePath)) + fileName := filepath.Base(configFilePath) + branch := strings.TrimSuffix(fileName, filepath.Ext(configFilePath)) - return org, repo, branch, nil + return &configFilePathElements{org, repo, branch, fileName}, nil } -func generateProwJobsFromConfigFile(configFilePath string) (*prowconfig.JobConfig, string, string, error) { +func generateProwJobsFromConfigFile(configFilePath string) (*prowconfig.JobConfig, *configFilePathElements, error) { configSpec, err := readCiOperatorConfig(configFilePath) if err != nil { - return nil, "", "", err + return nil, nil, err } - org, repo, branch, err := extractRepoElementsFromPath(configFilePath) + repoInfo, err := extractRepoElementsFromPath(configFilePath) if err != nil { - return nil, "", "", err + return nil, nil, err } + jobConfig := generateJobs(configSpec, repoInfo) - jobConfig := generateJobs(configSpec, org, repo, branch) - - return jobConfig, org, repo, nil + return jobConfig, repoInfo, nil } // Given a JobConfig and a target directory, write the Prow job configuration @@ -272,6 +280,11 @@ func writeJobsIntoComponentDirectory(jobDir, org, repo string, jobConfig *prowco return nil } +func isConfigFile(path string, info os.FileInfo) bool { + extension := filepath.Ext(path) + return !info.IsDir() && (extension == ".yaml" || extension == ".yml" || extension == ".json") +} + // Iterate over all ci-operator config files under a given path and generate a // Prow job configuration files for each one under a different path, mimicking // the directory structure. @@ -281,14 +294,14 @@ func generateJobsFromDirectory(configDir, jobDir, jobFile string) error { fmt.Fprintf(os.Stderr, "Error encontered while generating Prow job config: %v\n", err) return err } - if !info.IsDir() && filepath.Ext(path) == ".json" { - jobConfig, org, repo, err := generateProwJobsFromConfigFile(path) + if isConfigFile(path, info) { + jobConfig, repoInfo, err := generateProwJobsFromConfigFile(path) if err != nil { return err } if len(jobDir) > 0 { - if err = writeJobsIntoComponentDirectory(jobDir, org, repo, jobConfig); err != nil { + if err = writeJobsIntoComponentDirectory(jobDir, repoInfo.org, repoInfo.repo, jobConfig); err != nil { return err } } else if len(jobFile) > 0 { @@ -406,7 +419,7 @@ func main() { } if len(opt.fromFile) > 0 { - jobConfig, org, repo, err := generateProwJobsFromConfigFile(opt.fromFile) + jobConfig, repoInfo, err := generateProwJobsFromConfigFile(opt.fromFile) if err != nil { fmt.Fprintf(os.Stderr, "failed to generate jobs from '%s' (%v)\n", opt.fromFile, err) os.Exit(1) @@ -417,7 +430,7 @@ func main() { os.Exit(1) } } else { // from file to directory - if err := writeJobsIntoComponentDirectory(opt.toDir, org, repo, jobConfig); err != nil { + if err := writeJobsIntoComponentDirectory(opt.toDir, repoInfo.org, repoInfo.repo, jobConfig); err != nil { fmt.Fprintf(os.Stderr, "failed to write jobs to '%s' (%v)\n", opt.toDir, err) os.Exit(1) } diff --git a/cmd/ci-operator-prowgen/main_test.go b/cmd/ci-operator-prowgen/main_test.go index 9dfa27a5..907bc8ee 100644 --- a/cmd/ci-operator-prowgen/main_test.go +++ b/cmd/ci-operator-prowgen/main_test.go @@ -22,7 +22,7 @@ func TestGeneratePodSpec(t *testing.T) { tests := []struct { org string repo string - branch string + configFile string target string additionalArgs []string @@ -31,7 +31,7 @@ func TestGeneratePodSpec(t *testing.T) { { org: "organization", repo: "repo", - branch: "branch", + configFile: "config.json", target: "target", additionalArgs: []string{}, @@ -48,7 +48,7 @@ func TestGeneratePodSpec(t *testing.T) { LocalObjectReference: kubeapi.LocalObjectReference{ Name: "ci-operator-organization-repo", }, - Key: "branch.json", + Key: "config.json", }, }, }}, @@ -58,7 +58,7 @@ func TestGeneratePodSpec(t *testing.T) { { org: "organization", repo: "repo", - branch: "branch", + configFile: "config.yml", target: "target", additionalArgs: []string{"--promote", "something"}, @@ -75,7 +75,7 @@ func TestGeneratePodSpec(t *testing.T) { LocalObjectReference: kubeapi.LocalObjectReference{ Name: "ci-operator-organization-repo", }, - Key: "branch.json", + Key: "config.yml", }, }, }}, @@ -87,9 +87,9 @@ func TestGeneratePodSpec(t *testing.T) { for _, tc := range tests { var podSpec *kubeapi.PodSpec if len(tc.additionalArgs) == 0 { - podSpec = generatePodSpec(tc.org, tc.repo, tc.branch, tc.target) + podSpec = generatePodSpec(tc.org, tc.repo, tc.configFile, tc.target) } else { - podSpec = generatePodSpec(tc.org, tc.repo, tc.branch, tc.target, tc.additionalArgs...) + podSpec = generatePodSpec(tc.org, tc.repo, tc.configFile, tc.target, tc.additionalArgs...) } if !equality.Semantic.DeepEqual(podSpec, tc.expected) { t.Errorf("expected PodSpec diff:\n%s", diff.ObjectDiff(tc.expected, podSpec)) @@ -101,35 +101,29 @@ func TestGeneratePresubmitForTest(t *testing.T) { tests := []struct { name string target string - org string - repo string - branch string + repoInfo *configFilePathElements expected *prowconfig.Presubmit - }{ - { - name: "testname", - target: "target", - org: "org", - repo: "repo", - branch: "branch", - - expected: &prowconfig.Presubmit{ - Agent: "kubernetes", - AlwaysRun: true, - Brancher: prowconfig.Brancher{Branches: []string{"branch"}}, - Context: "ci/prow/testname", - Name: "pull-ci-org-repo-branch-testname", - RerunCommand: "/test testname", - Trigger: `((?m)^/test( all| testname),?(\\s+|$))`, - UtilityConfig: prowconfig.UtilityConfig{ - DecorationConfig: &prowkube.DecorationConfig{SkipCloning: true}, - Decorate: true, - }, + }{{ + name: "testname", + target: "target", + repoInfo: &configFilePathElements{org: "org", repo: "repo", branch: "branch"}, + + expected: &prowconfig.Presubmit{ + Agent: "kubernetes", + AlwaysRun: true, + Brancher: prowconfig.Brancher{Branches: []string{"branch"}}, + Context: "ci/prow/testname", + Name: "pull-ci-org-repo-branch-testname", + RerunCommand: "/test testname", + Trigger: `((?m)^/test( all| testname),?(\\s+|$))`, + UtilityConfig: prowconfig.UtilityConfig{ + DecorationConfig: &prowkube.DecorationConfig{SkipCloning: true}, + Decorate: true, }, }, - } + }} for _, tc := range tests { - presubmit := generatePresubmitForTest(testDescription{tc.name, tc.target}, tc.org, tc.repo, tc.branch) + presubmit := generatePresubmitForTest(testDescription{tc.name, tc.target}, tc.repoInfo) presubmit.Spec = nil // tested in generatePodSpec if !equality.Semantic.DeepEqual(presubmit, tc.expected) { @@ -142,20 +136,21 @@ func TestGeneratePostSubmitForTest(t *testing.T) { tests := []struct { name string target string - org string - repo string - branch string + repoInfo *configFilePathElements labels map[string]string additionalArgs []string expected *prowconfig.Postsubmit }{ { - name: "name", - target: "target", - org: "organization", - repo: "repository", - branch: "branch", + name: "name", + target: "target", + repoInfo: &configFilePathElements{ + org: "organization", + repo: "repository", + branch: "branch", + configFilename: "branch.yaml", + }, labels: map[string]string{}, additionalArgs: []string{}, @@ -170,11 +165,14 @@ func TestGeneratePostSubmitForTest(t *testing.T) { }, }, { - name: "name", - target: "target", - org: "organization", - repo: "repository", - branch: "branch", + name: "name", + target: "target", + repoInfo: &configFilePathElements{ + org: "organization", + repo: "repository", + branch: "branch", + configFilename: "branch.yaml", + }, labels: map[string]string{}, additionalArgs: []string{"--promote", "additionalArg"}, @@ -187,13 +185,15 @@ func TestGeneratePostSubmitForTest(t *testing.T) { Decorate: true, }, }, - }, - { - name: "Name", - target: "Target", - org: "Organization", - repo: "Repository", - branch: "Branch", + }, { + name: "Name", + target: "Target", + repoInfo: &configFilePathElements{ + org: "Organization", + repo: "Repository", + branch: "Branch", + configFilename: "config.yaml", + }, labels: map[string]string{"artifacts": "images"}, additionalArgs: []string{"--promote", "additionalArg"}, @@ -213,9 +213,9 @@ func TestGeneratePostSubmitForTest(t *testing.T) { var postsubmit *prowconfig.Postsubmit if len(tc.additionalArgs) == 0 { - postsubmit = generatePostsubmitForTest(testDescription{tc.name, tc.target}, tc.org, tc.repo, tc.branch, tc.labels) + postsubmit = generatePostsubmitForTest(testDescription{tc.name, tc.target}, tc.repoInfo, tc.labels) } else { - postsubmit = generatePostsubmitForTest(testDescription{tc.name, tc.target}, tc.org, tc.repo, tc.branch, tc.labels, tc.additionalArgs...) + postsubmit = generatePostsubmitForTest(testDescription{tc.name, tc.target}, tc.repoInfo, tc.labels, tc.additionalArgs...) // tests that additional args were propagated to the PodSpec if !equality.Semantic.DeepEqual(postsubmit.Spec.Containers[0].Args[2:], tc.additionalArgs) { t.Errorf("additional args not propagated to postsubmit:\n%s", diff.ObjectDiff(tc.additionalArgs, postsubmit.Spec.Containers[0].Args[2:])) @@ -232,11 +232,9 @@ func TestGeneratePostSubmitForTest(t *testing.T) { func TestGenerateJobs(t *testing.T) { tests := []struct { - id string - config *ciop.ReleaseBuildConfiguration - org string - repo string - branch string + id string + config *ciop.ReleaseBuildConfiguration + repoInfo *configFilePathElements expectedPresubmits map[string][]string expectedPostsubmits map[string][]string @@ -247,9 +245,12 @@ func TestGenerateJobs(t *testing.T) { config: &ciop.ReleaseBuildConfiguration{ Tests: []ciop.TestStepConfiguration{{As: "derTest"}, {As: "leTest"}}, }, - org: "organization", - repo: "repository", - branch: "branch", + repoInfo: &configFilePathElements{ + org: "organization", + repo: "repository", + branch: "branch", + configFilename: "konfig.yaml", + }, expected: &prowconfig.JobConfig{ Presubmits: map[string][]prowconfig.Presubmit{"organization/repository": { {Name: "pull-ci-organization-repository-branch-derTest"}, @@ -263,9 +264,12 @@ func TestGenerateJobs(t *testing.T) { Tests: []ciop.TestStepConfiguration{{As: "derTest"}, {As: "leTest"}}, Images: []ciop.ProjectDirectoryImageBuildStepConfiguration{{}}, }, - org: "organization", - repo: "repository", - branch: "branch", + repoInfo: &configFilePathElements{ + org: "organization", + repo: "repository", + branch: "branch", + configFilename: "konfig.yaml", + }, expected: &prowconfig.JobConfig{ Presubmits: map[string][]prowconfig.Presubmit{"organization/repository": { {Name: "pull-ci-organization-repository-branch-derTest"}, @@ -283,9 +287,12 @@ func TestGenerateJobs(t *testing.T) { Images: []ciop.ProjectDirectoryImageBuildStepConfiguration{{}}, PromotionConfiguration: &ciop.PromotionConfiguration{Namespace: "openshift"}, }, - org: "organization", - repo: "repository", - branch: "branch", + repoInfo: &configFilePathElements{ + org: "organization", + repo: "repository", + branch: "branch", + configFilename: "konfig.yaml", + }, expected: &prowconfig.JobConfig{ Presubmits: map[string][]prowconfig.Presubmit{"organization/repository": { {Name: "pull-ci-organization-repository-branch-images"}, @@ -302,9 +309,12 @@ func TestGenerateJobs(t *testing.T) { Images: []ciop.ProjectDirectoryImageBuildStepConfiguration{{}}, PromotionConfiguration: &ciop.PromotionConfiguration{Namespace: "ci"}, }, - org: "organization", - repo: "repository", - branch: "branch", + repoInfo: &configFilePathElements{ + org: "organization", + repo: "repository", + branch: "branch", + configFilename: "konfig.yaml", + }, expected: &prowconfig.JobConfig{ Presubmits: map[string][]prowconfig.Presubmit{"organization/repository": { {Name: "pull-ci-organization-repository-branch-images"}, @@ -322,9 +332,12 @@ func TestGenerateJobs(t *testing.T) { ReleaseTagConfiguration: &ciop.ReleaseTagConfiguration{Namespace: "openshift"}, }, }, - org: "organization", - repo: "repository", - branch: "branch", + repoInfo: &configFilePathElements{ + org: "organization", + repo: "repository", + branch: "branch", + configFilename: "konfig.yaml", + }, expected: &prowconfig.JobConfig{ Presubmits: map[string][]prowconfig.Presubmit{"organization/repository": { {Name: "pull-ci-organization-repository-branch-images"}, @@ -343,9 +356,12 @@ func TestGenerateJobs(t *testing.T) { ReleaseTagConfiguration: &ciop.ReleaseTagConfiguration{Namespace: "ci"}, }, }, - org: "organization", - repo: "repository", - branch: "branch", + repoInfo: &configFilePathElements{ + org: "organization", + repo: "repository", + branch: "branch", + configFilename: "konfig.yaml", + }, expected: &prowconfig.JobConfig{ Presubmits: map[string][]prowconfig.Presubmit{"organization/repository": { {Name: "pull-ci-organization-repository-branch-images"}, @@ -364,9 +380,12 @@ func TestGenerateJobs(t *testing.T) { }, PromotionConfiguration: &ciop.PromotionConfiguration{Namespace: "ci"}, }, - org: "organization", - repo: "repository", - branch: "branch", + repoInfo: &configFilePathElements{ + org: "organization", + repo: "repository", + branch: "branch", + configFilename: "konfig.yaml", + }, expected: &prowconfig.JobConfig{ Presubmits: map[string][]prowconfig.Presubmit{"organization/repository": { {Name: "pull-ci-organization-repository-branch-images"}, @@ -385,9 +404,12 @@ func TestGenerateJobs(t *testing.T) { }, PromotionConfiguration: &ciop.PromotionConfiguration{Namespace: "openshift"}, }, - org: "organization", - repo: "repository", - branch: "branch", + repoInfo: &configFilePathElements{ + org: "organization", + repo: "repository", + branch: "branch", + configFilename: "konfig.yaml", + }, expected: &prowconfig.JobConfig{ Presubmits: map[string][]prowconfig.Presubmit{"organization/repository": { {Name: "pull-ci-organization-repository-branch-images"}, @@ -402,7 +424,7 @@ func TestGenerateJobs(t *testing.T) { log.SetOutput(ioutil.Discard) for _, tc := range tests { - jobConfig := generateJobs(tc.config, tc.org, tc.repo, tc.branch) + jobConfig := generateJobs(tc.config, tc.repoInfo) prune(jobConfig) // prune the fields that are tested in TestGeneratePre/PostsubmitForTest @@ -437,35 +459,39 @@ func prune(jobConfig *prowconfig.JobConfig) { func TestExtractRepoElementsFromPath(t *testing.T) { testCases := []struct { - path string - expectedOrg string - expectedRepo string - expectedBranch string - expectedError bool + path string + expectedOrg string + expectedRepo string + expectedBranch string + expectedConfigFilename string + expectedError bool }{ - {"../../ci-operator/openshift/component/master.json", "openshift", "component", "master", false}, - {"master.json", "", "", "", true}, - {"dir/master.json", "", "", "", true}, + {"../../ci-operator/openshift/component/master.yaml", "openshift", "component", "master", "master.yaml", false}, + {"master.yaml", "", "", "", "", true}, + {"dir/master.yaml", "", "", "", "", true}, } for _, tc := range testCases { t.Run(tc.path, func(t *testing.T) { - org, repo, branch, err := extractRepoElementsFromPath(tc.path) + repoInfo, err := extractRepoElementsFromPath(tc.path) if !tc.expectedError { if err != nil { t.Errorf("returned unexpected error '%v", err) } - if org != tc.expectedOrg { - t.Errorf("org extracted incorrectly: got '%s', expected '%s'", org, tc.expectedOrg) + if repoInfo.org != tc.expectedOrg { + t.Errorf("org extracted incorrectly: got '%s', expected '%s'", repoInfo.org, tc.expectedOrg) } - if repo != tc.expectedRepo { - t.Errorf("repo extracted incorrectly: got '%s', expected '%s'", repo, tc.expectedRepo) + if repoInfo.repo != tc.expectedRepo { + t.Errorf("repo extracted incorrectly: got '%s', expected '%s'", repoInfo.repo, tc.expectedRepo) } - if branch != tc.expectedBranch { - t.Errorf("branch extracted incorrectly: got '%s', expected '%s'", branch, tc.expectedBranch) + if repoInfo.branch != tc.expectedBranch { + t.Errorf("branch extracted incorrectly: got '%s', expected '%s'", repoInfo.branch, tc.expectedBranch) + } + if repoInfo.configFilename != tc.expectedConfigFilename { + t.Errorf("configFilename extracted incorrectly: got '%s', expected '%s'", repoInfo.configFilename, tc.expectedConfigFilename) } } else { // expected error if err == nil { - t.Errorf("expected to return error, got org=%s repo=%s branch=%s instead", org, repo, branch) + t.Errorf("expected to return error, got org=%v", repoInfo) } } }) @@ -593,7 +619,7 @@ func TestMergeJobConfig(t *testing.T) { } } -func prepareInputs(org, component, branch string, configJSON, prowConfigYAML []byte) (string, string, string, error) { +func prepareInputs(org, component, branch string, configYAML, prowConfigYAML []byte) (string, string, string, error) { dir, err := ioutil.TempDir("", "prowgen-test") if err != nil { return "", "", "", err @@ -604,10 +630,10 @@ func prepareInputs(org, component, branch string, configJSON, prowConfigYAML []b return "", "", dir, err } - fullConfigPath := filepath.Join(workDir, fmt.Sprintf("%s.json", branch)) + fullConfigPath := filepath.Join(workDir, fmt.Sprintf("%s.yaml", branch)) fullProwConfigPath := filepath.Join(workDir, "jobs.yaml") - if err = ioutil.WriteFile(fullConfigPath, configJSON, 0664); err != nil { + if err = ioutil.WriteFile(fullConfigPath, configYAML, 0664); err != nil { return "", "", dir, err } if err = ioutil.WriteFile(fullProwConfigPath, prowConfigYAML, 0664); err != nil { @@ -619,18 +645,20 @@ func prepareInputs(org, component, branch string, configJSON, prowConfigYAML []b func TestFromCIOperatorConfigToProwYaml(t *testing.T) { tests := []struct { + id string org string component string branch string - configJSON []byte + configYAML []byte prowOldYAML []byte prowExpectedYAML []byte }{ { + id: "one test and images, no previous jobs. Expect test presubmit + pre/post submit images jobs", org: "super", component: "duper", branch: "branch", - configJSON: []byte(`{ + configYAML: []byte(`{ "tag_specification": { "cluster": "https://api.ci.openshift.org", "namespace": "openshift", "name": "origin-v3.11", "tag": "" }, @@ -668,7 +696,7 @@ func TestFromCIOperatorConfigToProwYaml(t *testing.T) { - name: CONFIG_SPEC valueFrom: configMapKeyRef: - key: branch.json + key: branch.yaml name: ci-operator-super-duper image: ci-operator:latest name: "" @@ -696,7 +724,7 @@ presubmits: - name: CONFIG_SPEC valueFrom: configMapKeyRef: - key: branch.json + key: branch.yaml name: ci-operator-super-duper image: ci-operator:latest name: "" @@ -723,7 +751,7 @@ presubmits: - name: CONFIG_SPEC valueFrom: configMapKeyRef: - key: branch.json + key: branch.yaml name: ci-operator-super-duper image: ci-operator:latest name: "" @@ -731,10 +759,11 @@ presubmits: serviceAccountName: ci-operator trigger: ((?m)^/test( all| images),?(\\s+|$)) `)}, { + id: "One test and images, one existing job. Expect one presubmit, pre/post submit images jobs. Existing job should not be changed.", org: "super", component: "duper", branch: "branch", - configJSON: []byte(`{ + configYAML: []byte(`{ "tag_specification": { "cluster": "https://api.ci.openshift.org", "namespace": "openshift", "name": "origin-v3.11", "tag": "" }, @@ -768,7 +797,7 @@ presubmits: - name: CONFIG_SPEC valueFrom: configMapKeyRef: - key: branch.json + key: branch.yaml name: ci-operator-super-duper image: ci-operator:latest name: "" @@ -797,15 +826,174 @@ presubmits: - name: CONFIG_SPEC valueFrom: configMapKeyRef: - key: branch.json + key: branch.yaml + name: ci-operator-super-duper + image: ci-operator:latest + name: "" + resources: {} + serviceAccountName: ci-operator + - agent: kubernetes + branches: + - branch + decorate: true + name: branch-ci-super-duper-branch-do-not-overwrite + skip_cloning: true + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --target=unit + command: + - ci-operator + env: + - name: CONFIG_SPEC + valueFrom: + configMapKeyRef: + key: branch.yaml + name: ci-operator-super-duper + image: ci-operator:latest + name: "" + resources: {} + serviceAccountName: ci-operator +presubmits: + super/duper: + - agent: kubernetes + always_run: true + branches: + - branch + context: ci/prow/unit + decorate: true + name: pull-ci-super-duper-branch-unit + rerun_command: /test unit + skip_cloning: true + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --target=unit + command: + - ci-operator + env: + - name: CONFIG_SPEC + valueFrom: + configMapKeyRef: + key: branch.yaml name: ci-operator-super-duper image: ci-operator:latest name: "" resources: {} serviceAccountName: ci-operator + trigger: ((?m)^/test( all| unit),?(\\s+|$)) - agent: kubernetes + always_run: true branches: - branch + context: ci/prow/images + decorate: true + name: pull-ci-super-duper-branch-images + rerun_command: /test images + skip_cloning: true + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --target=[images] + command: + - ci-operator + env: + - name: CONFIG_SPEC + valueFrom: + configMapKeyRef: + key: branch.yaml + name: ci-operator-super-duper + image: ci-operator:latest + name: "" + resources: {} + serviceAccountName: ci-operator + trigger: ((?m)^/test( all| images),?(\\s+|$)) +`), + }, { + id: "Input is YAML and it is correctly processed", + org: "super", + component: "duper", + branch: "branch", + configYAML: []byte(`base_images: + base: + cluster: https://api.ci.openshift.org + name: origin-v3.11 + namespace: openshift + tag: base +images: +- from: base + to: service-serving-cert-signer +tag_specification: + cluster: https://api.ci.openshift.org + name: origin-v3.11 + namespace: openshift + tag: '' +test_base_image: + cluster: https://api.ci.openshift.org + name: release + namespace: openshift + tag: golang-1.10 +tests: +- as: unit + commands: make test-unit + from: src +`), + prowOldYAML: []byte(`postsubmits: + super/duper: + - agent: kubernetes + decorate: true + name: branch-ci-super-duper-branch-do-not-overwrite + skip_cloning: true + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --target=unit + command: + - ci-operator + env: + - name: CONFIG_SPEC + valueFrom: + configMapKeyRef: + key: branch.yaml + name: ci-operator-super-duper + image: ci-operator:latest + name: "" + resources: {} + serviceAccountName: ci-operator +`), + prowExpectedYAML: []byte(`postsubmits: + super/duper: + - agent: kubernetes + branches: + - branch + decorate: true + labels: + artifacts: images + name: branch-ci-super-duper-branch-images + skip_cloning: true + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --target=[images] + - --promote + command: + - ci-operator + env: + - name: CONFIG_SPEC + valueFrom: + configMapKeyRef: + key: branch.yaml + name: ci-operator-super-duper + image: ci-operator:latest + name: "" + resources: {} + serviceAccountName: ci-operator + - agent: kubernetes decorate: true name: branch-ci-super-duper-branch-do-not-overwrite skip_cloning: true @@ -820,7 +1008,7 @@ presubmits: - name: CONFIG_SPEC valueFrom: configMapKeyRef: - key: branch.json + key: branch.yaml name: ci-operator-super-duper image: ci-operator:latest name: "" @@ -848,7 +1036,7 @@ presubmits: - name: CONFIG_SPEC valueFrom: configMapKeyRef: - key: branch.json + key: branch.yaml name: ci-operator-super-duper image: ci-operator:latest name: "" @@ -875,7 +1063,7 @@ presubmits: - name: CONFIG_SPEC valueFrom: configMapKeyRef: - key: branch.json + key: branch.yaml name: ci-operator-super-duper image: ci-operator:latest name: "" @@ -886,7 +1074,7 @@ presubmits: }, } for _, tc := range tests { - configPath, prowJobsPath, tempDir, err := prepareInputs(tc.org, tc.component, tc.branch, tc.configJSON, tc.prowOldYAML) + configPath, prowJobsPath, tempDir, err := prepareInputs(tc.org, tc.component, tc.branch, tc.configYAML, tc.prowOldYAML) if tempDir != "" { defer os.RemoveAll(tempDir) } @@ -895,7 +1083,7 @@ presubmits: continue } - jobConfig, _, _, err := generateProwJobsFromConfigFile(configPath) + jobConfig, _, err := generateProwJobsFromConfigFile(configPath) if err != nil { t.Errorf("Unexpected error: %v", err) continue