Migrate openshift/origin/master to build01

GENERATOR.md

@@ -48,7 +48,6 @@ following fields will not be overwritten if they are already present:
  - `optional`
  - `max_concurrency`
  - `skip_report`
- - `cluster`
 
 ## Postsubmits
 

cmd/ci-operator-prowgen/main.go

@@ -14,14 +14,14 @@ import (
 	"github.com/sirupsen/logrus"
 	kubeapi "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/apimachinery/pkg/util/sets"
 	v1 "k8s.io/test-infra/prow/apis/prowjobs/v1"
 	prowconfig "k8s.io/test-infra/prow/config"
 
 	cioperatorapi "github.com/openshift/ci-tools/pkg/api"
 	"github.com/openshift/ci-tools/pkg/config"
 	"github.com/openshift/ci-tools/pkg/jobconfig"
 	jc "github.com/openshift/ci-tools/pkg/jobconfig"
+	"github.com/openshift/ci-tools/pkg/migrate"
 	"github.com/openshift/ci-tools/pkg/promotion"
 )
 
@@ -139,7 +139,7 @@ func generatePodSpec(info *prowgenInfo, secrets []*cioperatorapi.Secret) *kubeap
 		ReadOnly:  true,
 	}}
 
-	if migrated(info.Org, info.Repo) {
+	if migrate.Migrated(info.Org, info.Repo, info.Branch) {
 		volumeMounts = []kubeapi.VolumeMount{
 			{
 				Name:      sentryDsnMountName,
@@ -166,7 +166,7 @@ func generatePodSpec(info *prowgenInfo, secrets []*cioperatorapi.Secret) *kubeap
 		},
 	}}
 
-	if migrated(info.Org, info.Repo) {
+	if migrate.Migrated(info.Org, info.Repo, info.Branch) {
 		volumes = []kubeapi.Volume{
 			{
 				Name: sentryDsnMountName,
@@ -263,7 +263,7 @@ func generateCiOperatorPodSpec(info *prowgenInfo, secrets []*cioperatorapi.Secre
 		fmt.Sprintf("--branch=%s", info.Branch),
 	}, additionalArgs...)
 
-	if migrated(info.Org, info.Repo) {
+	if migrate.Migrated(info.Org, info.Repo, info.Branch) {
 		ret.Containers[0].Args = append([]string{
 			"--give-pr-author-access-to-namespace=true",
 			"--artifact-dir=$(ARTIFACTS)",
@@ -395,7 +395,7 @@ func generatePodSpecOthers(info *prowgenInfo, release string, test *cioperatorap
 	}
 	// TODO expose boskos (behind an oauth proxy) so it can be used by build clusters
 	if needsLeaseServer {
-		if migrated(info.Org, info.Repo) {
+		if migrate.Migrated(info.Org, info.Repo, info.Branch) {
 			container.Args = append(container.Args, "--lease-server=https://boskos-ci.svc.ci.openshift.org")
 			container.Args = append(container.Args, "--lease-server-username=ci")
 			container.Args = append(container.Args, "--lease-server-password-file=/etc/boskos/password")
@@ -468,14 +468,6 @@ func generatePodSpecOthers(info *prowgenInfo, release string, test *cioperatorap
 	return podSpec
 }
 
-var (
-	migratedRepos = sets.NewString()
-)
-
-func migrated(org, repo string) bool {
-	return migratedRepos.Has(fmt.Sprintf("%s/%s", org, repo))
-}
-
 func generatePodSpecRandom(info *prowgenInfo, test *cioperatorapi.TestStepConfiguration) *kubeapi.PodSpec {
 	podSpec := generatePodSpec(info, test.Secrets)
 	for _, p := range openshiftInstallerRandomProfiles {

cmd/config-migrator/README.md

@@ -0,0 +1,32 @@
+# config-migrator
+
+This is a temporary tool helping us migrating repos to build cluster(s).
+
+No image needs to be built out of this tool and we run it locally.
+
+## Steps for migrating repos
+
+* modify the `Migrated(org, repo string)` function to allow the repos that should be migrated.
+
+```bash
+$ make install
+$ cd /path_to_release_repo
+$ config-migrator --config-dir ./ci-operator/config/
+### git add . then commit
+$ ci-operator-prowgen --from-dir ./ci-operator/config/ --to-dir ./ci-operator/jobs/
+### git add . then commit and push
+
+### See detailed steps in example PR: https://github.com/openshift/release/pull/6986/commits
+
+```
+
+Create a PR (PR1) based on the pushed branch, and watch the rehearsal.
+If no job is with
+
+If some rehearsal is broken, fix it and iterate the above steps.
+
+If all passed or [Straw-man approach applied](https://issues.redhat.com/browse/DPTP-684?focusedCommentId=13861610&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-13861610),
+
+* push the latest code for this repo, and create a PR (PR2)
+* merge PR2 (with breaking changes) and PR1 (fixing the breaking changes)
+

cmd/config-migrator/main.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	"github.com/getlantern/deepcopy"
+	"github.com/sirupsen/logrus"
+
+	"github.com/openshift/ci-tools/pkg/api"
+
+	"github.com/openshift/ci-tools/pkg/config"
+	"github.com/openshift/ci-tools/pkg/migrate"
+)
+
+type options struct {
+	ConfigDir string
+}
+
+func gatherOptions() options {
+	o := options{}
+	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+	fs.StringVar(&o.ConfigDir, "config-dir", "", "Path to CI Operator configuration directory.")
+	if err := fs.Parse(os.Args[1:]); err != nil {
+		logrus.WithError(err).Fatal("could not parse input")
+	}
+	return o
+}
+
+func main() {
+	o := gatherOptions()
+
+	var toCommit []config.DataWithInfo
+	if err := config.OperateOnCIOperatorConfigDir(o.ConfigDir, func(configuration *api.ReleaseBuildConfiguration, info *config.Info) error {
+		for _, output := range generateMigratedConfigs(config.DataWithInfo{Configuration: *configuration, Info: *info}) {
+			// we are walking the config so we need to commit once we're done
+			toCommit = append(toCommit, output)
+		}
+
+		return nil
+	}); err != nil {
+		logrus.WithError(err).Fatal("Could not branch configurations.")
+	}
+
+	var failed bool
+	for _, output := range toCommit {
+		if err := output.CommitTo(o.ConfigDir); err != nil {
+			failed = true
+		}
+	}
+	if failed {
+		logrus.Fatal("Failed to commit configuration to disk.")
+	}
+}
+
+func generateMigratedConfigs(input config.DataWithInfo) []config.DataWithInfo {
+
+	if !migrate.Migrated(input.Info.Org, input.Info.Repo, input.Info.Branch) {
+		logrus.Debugf("%s/%s is not migrated", input.Info.Org, input.Info.Repo)
+		return nil
+	}
+	logrus.Infof("%s/%s is migrated", input.Info.Org, input.Info.Repo)
+
+	var output []config.DataWithInfo
+	input.Logger().Info("Migrating configuration.")
+	currentConfig := input.Configuration
+
+	var futureConfig api.ReleaseBuildConfiguration
+	if err := deepcopy.Copy(&futureConfig, &currentConfig); err != nil {
+		input.Logger().WithError(err).Error("failed to copy input CI Operator configuration")
+		return nil
+	}
+
+	var newBaseImages map[string]api.ImageStreamTagReference
+	for k, baseImage := range futureConfig.BaseImages {
+		if newBaseImages == nil {
+			newBaseImages = map[string]api.ImageStreamTagReference{}
+		}
+		if baseImage.Cluster == "" {
+			baseImage.Cluster = migrate.ProwClusterURL
+		}
+		newBaseImages[k] = baseImage
+	}
+	futureConfig.BaseImages = newBaseImages
+
+	if futureConfig.ReleaseTagConfiguration != nil && futureConfig.ReleaseTagConfiguration.Cluster == "" {
+		futureConfig.ReleaseTagConfiguration.Cluster = migrate.ProwClusterURL
+	}
+
+	if futureConfig.BuildRootImage != nil && futureConfig.BuildRootImage.ImageStreamTagReference != nil && futureConfig.BuildRootImage.ImageStreamTagReference.Cluster == "" {
+		futureConfig.BuildRootImage.ImageStreamTagReference.Cluster = migrate.ProwClusterURL
+	}
+
+	// this config will promote to the new location on the release branch
+	output = append(output, config.DataWithInfo{Configuration: futureConfig, Info: input.Info})
+	return output
+}

pkg/jobconfig/files.go

@@ -415,7 +415,6 @@ func mergePresubmits(old, new *prowconfig.Presubmit) prowconfig.Presubmit {
 	merged.Optional = old.Optional
 	merged.MaxConcurrency = old.MaxConcurrency
 	merged.SkipReport = old.SkipReport
-	merged.Cluster = old.Cluster
 
 	return merged
 }
@@ -427,7 +426,6 @@ func mergePostsubmits(old, new *prowconfig.Postsubmit) prowconfig.Postsubmit {
 	merged := *new
 
 	merged.MaxConcurrency = old.MaxConcurrency
-	merged.Cluster = old.Cluster
 
 	return merged
 }
@@ -439,7 +437,6 @@ func mergePeriodics(old, new *prowconfig.Periodic) prowconfig.Periodic {
 	merged := *new
 
 	merged.MaxConcurrency = old.MaxConcurrency
-	merged.Cluster = old.Cluster
 
 	return merged
 }

pkg/migrate/migrate.go

@@ -0,0 +1,23 @@
+package migrate
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+const (
+	ProwClusterURL = "https://api.ci.openshift.org"
+)
+
+var (
+	migratedRepos = sets.NewString(
+		"openshift/origin/master",
+	)
+)
+
+func Migrated(org, repo, branch string) bool {
+	// gradually, we can add regex
+	// eventually, we will return true without any check
+	return migratedRepos.Has(fmt.Sprintf("%s/%s/%s", org, repo, branch))
+}

pkg/migrate/migrate_test.go

@@ -0,0 +1,39 @@
+package migrate
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestMigrated(t *testing.T) {
+	var testCases = []struct {
+		testName string
+		org      string
+		repo     string
+		branch   string
+		expected bool
+	}{
+		{
+			testName: "openshift/origin/master is migrated",
+			org:      "openshift",
+			repo:     "origin",
+			branch:   "master",
+			expected: true,
+		},
+		{
+			testName: "openshift/some-repo/4.2 is NOT migrated",
+			org:      "openshift",
+			repo:     "some-repo",
+			branch:   "4.2",
+			expected: false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.testName, func(t *testing.T) {
+			if actual, expected := Migrated(testCase.org, testCase.repo, testCase.branch), testCase.expected; !reflect.DeepEqual(actual, expected) {
+				t.Errorf("%s: got incorrect result '%t', expecting '%t'", testCase.testName, actual, expected)
+			}
+		})
+	}
+}

test/repo-init-integration/expected/ci-operator/jobs/openshift/origin/openshift-origin-master-postsubmits.yaml

@@ -15,6 +15,8 @@ postsubmits:
         - --artifact-dir=$(ARTIFACTS)
         - --branch=master
         - --give-pr-author-access-to-namespace=true
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --kubeconfig=/etc/apici/kubeconfig
         - --org=openshift
         - --promote
         - --repo=origin
@@ -37,11 +39,26 @@ postsubmits:
           requests:
             cpu: 10m
         volumeMounts:
+        - mountPath: /etc/apici
+          name: apici-ci-operator-credentials
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
         - mountPath: /etc/sentry-dsn
           name: sentry-dsn
           readOnly: true
       serviceAccountName: ci-operator
       volumes:
+      - name: apici-ci-operator-credentials
+        secret:
+          items:
+          - key: sa.ci-operator.apici.config
+            path: kubeconfig
+          secretName: apici-ci-operator-credentials
+      - name: pull-secret
+        secret:
+          secretName: regcred
       - name: sentry-dsn
         secret:
           secretName: sentry-dsn