From 5789740fc41a04eb8ececf2592ab71c83b2747d7 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Thu, 7 Mar 2019 00:12:04 -0500 Subject: [PATCH 1/3] Bump ci-operator API definitions for upgrade testing --- Gopkg.lock | 4 +- .../openshift/ci-operator/CONFIGURATION.md | 22 +- .../ci-operator/cmd/ci-operator/main.go | 11 +- .../openshift/ci-operator/pkg/api/config.go | 6 +- .../ci-operator/pkg/api/parameters.go | 116 +++++ .../parameters_test.go} | 34 +- .../openshift/ci-operator/pkg/api/types.go | 26 +- .../ci-operator/pkg/defaults/defaults.go | 48 +- .../ci-operator/pkg/steps/artifacts.go | 13 +- .../openshift/ci-operator/pkg/steps/pod.go | 29 +- .../pkg/steps/release/create_release.go | 353 ++++++++++++-- .../pkg/steps/{ => release}/promote.go | 2 +- .../pkg/steps/release/release_images.go | 318 ++++++++++++ .../ci-operator/pkg/steps/release_images.go | 459 ------------------ .../ci-operator/pkg/steps/rpm_injection.go | 2 +- .../openshift/ci-operator/pkg/steps/source.go | 9 +- .../ci-operator/pkg/steps/template.go | 137 +----- .../ci-operator/pkg/steps/write_params.go | 4 +- .../pkg/steps/write_params_test.go | 2 +- 19 files changed, 877 insertions(+), 718 deletions(-) create mode 100644 vendor/github.com/openshift/ci-operator/pkg/api/parameters.go rename vendor/github.com/openshift/ci-operator/pkg/{steps/template_test.go => api/parameters_test.go} (90%) rename vendor/github.com/openshift/ci-operator/pkg/steps/{ => release}/promote.go (99%) create mode 100644 vendor/github.com/openshift/ci-operator/pkg/steps/release/release_images.go delete mode 100644 vendor/github.com/openshift/ci-operator/pkg/steps/release_images.go diff --git a/Gopkg.lock b/Gopkg.lock index 453b5980..e44e254e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -365,11 +365,11 @@ [[projects]] branch = "master" - digest = "1:349e22a8ae3b122b27dc1f0bb88803a0d6bdb37e9117ae9160966b9738168111" + digest = "1:7829a61c97cb22bedf5858a94c04e17d94f79293743879a78df746b594c692b6" name = "github.com/openshift/ci-operator" packages = ["pkg/api"] pruneopts = "" - revision = "3ae51bc3f5d8874342611325573ec77adee8835f" + revision = "03fd080a27afe9918509a42ee3f797c4e1b1cb31" [[projects]] digest = "1:7da30d36edabf50fb5bc07f759991589c5ca72e75b3dd50fa58c57d94e96fd15" diff --git a/vendor/github.com/openshift/ci-operator/CONFIGURATION.md b/vendor/github.com/openshift/ci-operator/CONFIGURATION.md index 009f86d2..8c0c54fc 100644 --- a/vendor/github.com/openshift/ci-operator/CONFIGURATION.md +++ b/vendor/github.com/openshift/ci-operator/CONFIGURATION.md @@ -97,22 +97,12 @@ tag_specification: cluster: https://api.ci.openshift.org name: origin-v3.11 namespace: openshift - tag: '' tag_overrides: {} ``` -There are two primary modes for assembling a release: - - single `ImageStream`, multiple tags (`openshift/origin-v3.9:control-plane`) - - multiple `ImageStream`s, one tag (`openshift/origin-control-plane:v3.9`) - -The former works well for central control, the latter for distributed control: -when many disparate processes are publishing images at their own cadences, each -process can own its own `ImageStream` and coordination between processes can be -through coordination in `ImageStreamTag`s. When one process marshalls a release, -the process can push to one `ImageStream` under multiple `ImageStreamTags`. In -practice, the OpenShift releases are assembled using the former approach, while -non-release images like infrastructure tooling, _etc_, are assembled using the -latter. +The release tag specification points to an image stream containing multiple tags, +each of which references a single component by a well known name, e.g. +`openshift/origin-v3.9:control-plane`. ## `tag_specification.cluster` `cluster` is an optional cluster string (`host`, `host:port`, or `scheme://host:port`) @@ -120,16 +110,12 @@ to connect to for the `ImageStream`. The referenced OpenShift cluster must suppo anonymous access to retrieve `ImageStream`s, `ImageStreamTag`s, and `ImageStreamImage`s in the provided namespace. -## `tag_specification.tag` -`tag` is used to specify the single tag when multiple `ImageStreams` but one tag -are used to assemble a release. - ## `tag_specification.namespace` `namespace` determines the `Namespace` on the target cluster where release `ImageStreams` are located. ## `tag_specification.name` -`name` is the `ImageStream` name when a single `ImageStream` but multiple +`name` is the `ImageStream` name where a single `ImageStream` and multiple tags are used to assemble a release. ## `tag_specification.tag_overrides` diff --git a/vendor/github.com/openshift/ci-operator/cmd/ci-operator/main.go b/vendor/github.com/openshift/ci-operator/cmd/ci-operator/main.go index 75f44e26..12a8447d 100644 --- a/vendor/github.com/openshift/ci-operator/cmd/ci-operator/main.go +++ b/vendor/github.com/openshift/ci-operator/cmd/ci-operator/main.go @@ -842,10 +842,19 @@ func jobSpecFromGitRef(ref string) (*api.JobSpec, error) { if err != nil { return nil, fmt.Errorf("'git ls-remote %s %s' failed with '%s'", repo, parts[1], err) } - sha := strings.Split(strings.Split(string(out), "\n")[0], "\t")[0] + resolved := strings.Split(strings.Split(string(out), "\n")[0], "\t") + sha := resolved[0] if len(sha) == 0 { return nil, fmt.Errorf("ref '%s' does not point to any commit in '%s'", parts[1], parts[0]) } + // sanity check that regular refs are fully determined + if strings.HasPrefix(resolved[1], "refs/heads/") && !strings.HasPrefix(parts[1], "refs/heads/") { + if resolved[1] != ("refs/heads/" + parts[1]) { + trimmed := resolved[1][len("refs/heads/"):] + // we could fix this for the user, but better to require them to be explicit + return nil, fmt.Errorf("ref '%s' does not point to any commit in '%s' (did you mean '%s'?)", parts[1], parts[0], trimmed) + } + } log.Printf("Resolved %s to commit %s", ref, sha) return &api.JobSpec{Type: api.PeriodicJob, Job: "dev", Refs: &api.Refs{Org: prefix[0], Repo: prefix[1], BaseRef: parts[1], BaseSHA: sha}}, nil } diff --git a/vendor/github.com/openshift/ci-operator/pkg/api/config.go b/vendor/github.com/openshift/ci-operator/pkg/api/config.go index 8e14301b..5a8b9e84 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/api/config.go +++ b/vendor/github.com/openshift/ci-operator/pkg/api/config.go @@ -63,7 +63,7 @@ func validatePromotionWithTagSpec(promotion *PromotionConfiguration, tagSpec *Re validationErrors = append(validationErrors, fmt.Errorf("promotion: no namespace defined")) } if len(promotion.Name) == 0 && len(promotion.Tag) == 0 { - if len(tagSpec.Name) != 0 || len(tagSpec.Tag) != 0 { + if len(tagSpec.Name) != 0 { // will get defaulted, is ok } else { validationErrors = append(validationErrors, errors.New("promotion: no name or tag provided and could not derive defaults from tag_specification")) @@ -169,8 +169,8 @@ func validateReleaseTagConfiguration(fieldRoot string, input ReleaseTagConfigura validationErrors = append(validationErrors, fmt.Errorf("%s: no namespace defined", fieldRoot)) } - if len(input.Name) == 0 && len(input.Tag) == 0 { - validationErrors = append(validationErrors, fmt.Errorf("%s: no name or tag defined", fieldRoot)) + if len(input.Name) == 0 { + validationErrors = append(validationErrors, fmt.Errorf("%s: no name defined", fieldRoot)) } return validationErrors } diff --git a/vendor/github.com/openshift/ci-operator/pkg/api/parameters.go b/vendor/github.com/openshift/ci-operator/pkg/api/parameters.go new file mode 100644 index 00000000..bdb06d1b --- /dev/null +++ b/vendor/github.com/openshift/ci-operator/pkg/api/parameters.go @@ -0,0 +1,116 @@ +package api + +import ( + "fmt" + "os" + "sync" +) + +type DeferredParameters struct { + lock sync.Mutex + fns ParameterMap + values map[string]string + links map[string][]StepLink +} + +func NewDeferredParameters() *DeferredParameters { + return &DeferredParameters{ + fns: make(ParameterMap), + values: make(map[string]string), + links: make(map[string][]StepLink), + } +} + +func (p *DeferredParameters) Map() (map[string]string, error) { + p.lock.Lock() + defer p.lock.Unlock() + m := make(map[string]string) + for k, fn := range p.fns { + if v, ok := p.values[k]; ok { + m[k] = v + continue + } + v, err := fn() + if err != nil { + return nil, fmt.Errorf("could not lazily evaluate deferred parameter: %v", err) + } + p.values[k] = v + m[k] = v + } + return m, nil +} + +func (p *DeferredParameters) Set(name, value string) { + p.lock.Lock() + defer p.lock.Unlock() + if _, ok := p.fns[name]; ok { + return + } + if _, ok := p.values[name]; ok { + return + } + p.values[name] = value +} + +func (p *DeferredParameters) Add(name string, link StepLink, fn func() (string, error)) { + p.lock.Lock() + defer p.lock.Unlock() + p.fns[name] = fn + if link != nil { + p.links[name] = []StepLink{link} + } +} + +func (p *DeferredParameters) Has(name string) bool { + p.lock.Lock() + defer p.lock.Unlock() + _, ok := p.fns[name] + if ok { + return true + } + _, ok = os.LookupEnv(name) + return ok +} + +func (p *DeferredParameters) Links(name string) []StepLink { + p.lock.Lock() + defer p.lock.Unlock() + if _, ok := os.LookupEnv(name); ok { + return nil + } + return p.links[name] +} + +func (p *DeferredParameters) AllLinks() []StepLink { + p.lock.Lock() + defer p.lock.Unlock() + var links []StepLink + for name, v := range p.links { + if _, ok := os.LookupEnv(name); ok { + continue + } + links = append(links, v...) + } + return links +} + +func (p *DeferredParameters) Get(name string) (string, error) { + p.lock.Lock() + defer p.lock.Unlock() + if value, ok := p.values[name]; ok { + return value, nil + } + if value, ok := os.LookupEnv(name); ok { + p.values[name] = value + return value, nil + } + if fn, ok := p.fns[name]; ok { + value, err := fn() + if err != nil { + return "", fmt.Errorf("could not lazily evaluate deferred parameter: %v", err) + } + p.values[name] = value + return value, nil + } + return "", nil +} diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/template_test.go b/vendor/github.com/openshift/ci-operator/pkg/api/parameters_test.go similarity index 90% rename from vendor/github.com/openshift/ci-operator/pkg/steps/template_test.go rename to vendor/github.com/openshift/ci-operator/pkg/api/parameters_test.go index 1880d8e6..cd78e055 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/template_test.go +++ b/vendor/github.com/openshift/ci-operator/pkg/api/parameters_test.go @@ -1,14 +1,22 @@ -package steps +package api import ( "reflect" "testing" "k8s.io/apimachinery/pkg/util/diff" - - "github.com/openshift/ci-operator/pkg/api" ) +func someStepLink(as string) StepLink { + return ExternalImageLink(ImageStreamTagReference{ + Cluster: "cluster.com", + Namespace: "namespace", + Name: "name", + Tag: "tag", + As: as, + }) +} + func TestDeferredParametersAllLinks(t *testing.T) { var testCases = []struct { purpose string @@ -17,7 +25,7 @@ func TestDeferredParametersAllLinks(t *testing.T) { }{{ purpose: "AllLinks should return a slice with all links for all names", dp: &DeferredParameters{ - links: map[string][]api.StepLink{ + links: map[string][]StepLink{ "K1": {someStepLink("ONE"), someStepLink("TWO")}, "K2": {someStepLink("THREE")}, }, @@ -93,11 +101,11 @@ func TestDeferredParametersAddHasLinksGet(t *testing.T) { dp *DeferredParameters callAdd bool name string - link api.StepLink + link StepLink fn func() (string, error) expectedHas bool - expectedLinks []api.StepLink + expectedLinks []StepLink expectedGet string }{{ purpose: "After `Add(key, link, f)`: Has(key)->true, Links(key)->{link}, Get(key)->f()", @@ -108,7 +116,7 @@ func TestDeferredParametersAddHasLinksGet(t *testing.T) { fn: func() (string, error) { return "value", nil }, expectedHas: true, - expectedLinks: []api.StepLink{someStepLink("name")}, + expectedLinks: []StepLink{someStepLink("name")}, expectedGet: "value", }, { purpose: "Without Add(): Has(key)->false and Links(key)->nil", @@ -124,9 +132,9 @@ func TestDeferredParametersAddHasLinksGet(t *testing.T) { }, { purpose: "After `Add(key, new-link)` when `key` already present: Has(key)->true and Links(key)->{new-link}", dp: &DeferredParameters{ - fns: api.ParameterMap{"key": func() (string, error) { return "old", nil }}, + fns: ParameterMap{"key": func() (string, error) { return "old", nil }}, values: map[string]string{}, - links: map[string][]api.StepLink{"key": {someStepLink("old-link")}}, + links: map[string][]StepLink{"key": {someStepLink("old-link")}}, }, callAdd: true, name: "key", @@ -134,7 +142,7 @@ func TestDeferredParametersAddHasLinksGet(t *testing.T) { fn: func() (string, error) { return "new", nil }, expectedHas: true, - expectedLinks: []api.StepLink{someStepLink("new-link")}, + expectedLinks: []StepLink{someStepLink("new-link")}, expectedGet: "new", }} for _, tc := range testCases { @@ -175,9 +183,9 @@ func TestDeferredParametersGetSet(t *testing.T) { }, { purpose: "Existing key is not overwritten", dp: &DeferredParameters{ - fns: make(api.ParameterMap), + fns: make(ParameterMap), values: map[string]string{"key": "oldValue"}, - links: map[string][]api.StepLink{}, + links: map[string][]StepLink{}, }, name: "key", callSet: true, @@ -192,7 +200,7 @@ func TestDeferredParametersGetSet(t *testing.T) { "key": func() (string, error) { return "lazyValue", nil }, }, values: map[string]string{}, - links: map[string][]api.StepLink{}, + links: map[string][]StepLink{}, }, name: "key", callSet: true, diff --git a/vendor/github.com/openshift/ci-operator/pkg/api/types.go b/vendor/github.com/openshift/ci-operator/pkg/api/types.go index 549fc900..11c6606f 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/api/types.go +++ b/vendor/github.com/openshift/ci-operator/pkg/api/types.go @@ -36,7 +36,7 @@ type ReleaseBuildConfiguration struct { // the desired location of the contents of this repository in // Go. If specified the location of the repository we are // cloning from is ignored. - CanonicalGoRepository string `json:"canonical_go_repository"` + CanonicalGoRepository string `json:"canonical_go_repository,omitempty"` // Images describes the images that are built // baseImage the project as part of the release @@ -152,11 +152,9 @@ type ImageStreamTagReference struct { } // ReleaseTagConfiguration describes how a release is -// assembled from release artifacts. There are two primary modes, -// single stream, multiple tags (openshift/origin-v3.9:control-plane) -// on one stream, or multiple streams with one tag -// (openshift/origin-control-plane:v3.9). The former works well for -// central control, the latter for distributed control. +// assembled from release artifacts. A release image stream is a +// single stream with multiple tags (openshift/origin-v3.9:control-plane), +// each tag being a unique and well defined name for a component. type ReleaseTagConfiguration struct { // Cluster is an optional cluster string (host, host:port, or // scheme://host:port) to connect to for this image stream. The @@ -170,15 +168,10 @@ type ReleaseTagConfiguration struct { // job are tagged from. Namespace string `json:"namespace"` - // Name is an optional image stream name to use that - // contains all component tags. If specified, tag is - // ignored. + // Name is the image stream name to use that contains all + // component tags. Name string `json:"name"` - // Tag is the ImageStreamTag tagged in for each - // ImageStream in the above Namespace. - Tag string `json:"tag,omitempty"` - // NamePrefix is prepended to the final output image name // if specified. NamePrefix string `json:"name_prefix,omitempty"` @@ -402,6 +395,11 @@ type OpenshiftAnsibleUpgradeClusterTestConfiguration struct { // conformance tests. type OpenshiftInstallerClusterTestConfiguration struct { ClusterTestConfiguration `json:",inline"` + // If upgrade is true, RELEASE_IMAGE_INITIAL will be used as + // the initial payload and the installer image from that + // will be upgraded. The `run-upgrade-tests` function will be + // available for the commands. + Upgrade bool `json:"upgrade"` } // OpenshiftInstallerSrcClusterTestConfiguration describes a @@ -529,4 +527,6 @@ const ( RPMServeLocation = "/srv/repo" StableImageStream = "stable" + + ComponentFormatReplacement = "${component}" ) diff --git a/vendor/github.com/openshift/ci-operator/pkg/defaults/defaults.go b/vendor/github.com/openshift/ci-operator/pkg/defaults/defaults.go index dc0c0974..f636dffd 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/defaults/defaults.go +++ b/vendor/github.com/openshift/ci-operator/pkg/defaults/defaults.go @@ -94,14 +94,14 @@ func FromConfig( podClient = steps.NewPodClient(coreGetter, clusterConfig, coreGetter.RESTClient()) } - params := steps.NewDeferredParameters() + params := api.NewDeferredParameters() params.Add("JOB_NAME", nil, func() (string, error) { return jobSpec.Job, nil }) params.Add("JOB_NAME_HASH", nil, func() (string, error) { return fmt.Sprintf("%x", sha256.Sum256([]byte(jobSpec.Job)))[:5], nil }) params.Add("JOB_NAME_SAFE", nil, func() (string, error) { return strings.Replace(jobSpec.Job, "_", "-", -1), nil }) params.Add("NAMESPACE", nil, func() (string, error) { return jobSpec.Namespace, nil }) var imageStepLinks []api.StepLink - var releaseStep api.Step + var releaseStep, initialReleaseStep api.Step for _, rawStep := range stepConfigsForBuild(config, jobSpec) { var step api.Step var stepLinks []api.StepLink @@ -138,10 +138,16 @@ func FromConfig( if err != nil { return nil, nil, fmt.Errorf("unable to access release images on remote cluster: %v", err) } - step = steps.ReleaseImagesTagStep(*rawStep.ReleaseImagesTagStepConfiguration, srcClient, imageClient, routeGetter, configMapGetter, params, jobSpec) + step = release.ReleaseImagesTagStep(*rawStep.ReleaseImagesTagStepConfiguration, srcClient, imageClient, routeGetter, configMapGetter, params, jobSpec) stepLinks = append(stepLinks, step.Creates()...) - releaseStep = release.AssembleReleaseStep(*rawStep.ReleaseImagesTagStepConfiguration, config.Resources, podClient, imageClient, artifactDir, jobSpec) + releaseStep = release.AssembleReleaseStep(true, *rawStep.ReleaseImagesTagStepConfiguration, config.Resources, podClient, imageClient, artifactDir, jobSpec) + checkForFullyQualifiedStep(releaseStep, params) + + initialReleaseStep = release.AssembleReleaseStep(false, *rawStep.ReleaseImagesTagStepConfiguration, config.Resources, podClient, imageClient, artifactDir, jobSpec) + checkForFullyQualifiedStep(initialReleaseStep, params) + // initial release is always added + buildSteps = append(buildSteps, initialReleaseStep) } else if rawStep.TestStepConfiguration != nil { step = steps.TestStep(*rawStep.TestStepConfiguration, config.Resources, podClient, artifactDir, jobSpec) @@ -164,10 +170,9 @@ func FromConfig( } if releaseStep != nil { - releaseStep, _ = checkForFullyQualifiedStep(releaseStep, params) buildSteps = append(buildSteps, releaseStep) } else { - buildSteps = append(buildSteps, steps.StableImagesTagStep(imageClient, jobSpec)) + buildSteps = append(buildSteps, release.StableImagesTagStep(imageClient, jobSpec)) } buildSteps = append(buildSteps, steps.ImagesReadyStep(imageStepLinks)) @@ -184,7 +189,7 @@ func FromConfig( tags = append(tags, string(image.To)) } } - postSteps = append(postSteps, steps.PromotionStep(*cfg, tags, imageClient, imageClient, jobSpec)) + postSteps = append(postSteps, release.PromotionStep(*cfg, tags, imageClient, imageClient, jobSpec)) } return buildSteps, postSteps, nil @@ -193,7 +198,7 @@ func FromConfig( // checkForFullyQualifiedStep if all output parameters of this step are part of the // environment, replace the step with a shim that automatically provides those variables. // Returns true if the step was replaced. -func checkForFullyQualifiedStep(step api.Step, params *steps.DeferredParameters) (api.Step, bool) { +func checkForFullyQualifiedStep(step api.Step, params *api.DeferredParameters) (api.Step, bool) { provides, link := step.Provides() if values, ok := envHasAllParameters(provides); ok { @@ -361,25 +366,14 @@ func stepConfigsForBuild(config *api.ReleaseBuildConfiguration, jobSpec *api.Job image := &config.Images[i] buildSteps = append(buildSteps, api.StepConfiguration{ProjectDirectoryImageBuildStepConfiguration: image}) if config.ReleaseTagConfiguration != nil { - if len(config.ReleaseTagConfiguration.Name) > 0 { - buildSteps = append(buildSteps, api.StepConfiguration{OutputImageTagStepConfiguration: &api.OutputImageTagStepConfiguration{ - From: image.To, - To: api.ImageStreamTagReference{ - Name: fmt.Sprintf("%s%s", config.ReleaseTagConfiguration.NamePrefix, api.StableImageStream), - Tag: string(image.To), - }, - Optional: image.Optional, - }}) - } else { - buildSteps = append(buildSteps, api.StepConfiguration{OutputImageTagStepConfiguration: &api.OutputImageTagStepConfiguration{ - From: image.To, - To: api.ImageStreamTagReference{ - Name: string(image.To), - Tag: "ci", - }, - Optional: image.Optional, - }}) - } + buildSteps = append(buildSteps, api.StepConfiguration{OutputImageTagStepConfiguration: &api.OutputImageTagStepConfiguration{ + From: image.To, + To: api.ImageStreamTagReference{ + Name: fmt.Sprintf("%s%s", config.ReleaseTagConfiguration.NamePrefix, api.StableImageStream), + Tag: string(image.To), + }, + Optional: image.Optional, + }}) } else { buildSteps = append(buildSteps, api.StepConfiguration{OutputImageTagStepConfiguration: &api.OutputImageTagStepConfiguration{ From: image.To, diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/artifacts.go b/vendor/github.com/openshift/ci-operator/pkg/steps/artifacts.go index 2a2cc6dd..6108ff63 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/artifacts.go +++ b/vendor/github.com/openshift/ci-operator/pkg/steps/artifacts.go @@ -15,13 +15,14 @@ import ( "sync" "time" - "k8s.io/apimachinery/pkg/util/sets" + "github.com/golang/glog" coreapi "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes/scheme" coreclientset "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" @@ -182,7 +183,7 @@ type PodClient interface { } func copyArtifacts(podClient PodClient, into, ns, name, containerName string, paths []string) error { - log.Printf("Copying artifacts from %s into %s", name, into) + glog.V(4).Infof("Copying artifacts from %s into %s", name, into) var args []string for _, s := range paths { args = append(args, "-C", s, ".") @@ -253,7 +254,10 @@ func copyArtifacts(podClient PodClient, into, ns, name, containerName string, pa size += h.Size } - if size > 0 { + // If we're updating a substantial amount of artifacts, let the user know as a way to + // indicate why the step took a long amount of time. Conversely, if we just got a small + // number of files this is just noise and can be omitted to not distract from other steps. + if size > 1*1000*1000 { log.Printf("Copied %0.2fMi of artifacts from %s to %s", float64(size)/1000000, name, into) } @@ -651,6 +655,9 @@ func gatherContainerLogsOutput(podClient PodClient, artifactDir, namespace, podN if err != nil { return fmt.Errorf("could not list pod: %v", err) } + if len(list.Items) == 0 { + return nil + } pod := &list.Items[0] if pod.Annotations["ci-operator.openshift.io/save-container-logs"] != "true" { diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/pod.go b/vendor/github.com/openshift/ci-operator/pkg/steps/pod.go index 69e3934a..9b5a2550 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/pod.go +++ b/vendor/github.com/openshift/ci-operator/pkg/steps/pod.go @@ -21,7 +21,15 @@ import ( const testSecretName = "test-secret" const testSecretDefaultPath = "/usr/test-secrets" +// PodStepConfiguration allows other steps to reuse the pod launching and monitoring +// behavior without reimplementing function. It also enforces conventions like naming, +// directory structure, and input image format. More sophisticated reuse of launching +// pods should use RunPod which is more limited. type PodStepConfiguration struct { + // SkipLogs instructs the step to omit informational logs, such as when the pod is + // part of a larger step like release creation where displaying pod specific info + // is confusing to an end user. Failure logs are still printed. + SkipLogs bool As string From api.ImageStreamTagReference Commands string @@ -48,8 +56,9 @@ func (s *podStep) Inputs(ctx context.Context, dry bool) (api.InputDefinition, er } func (s *podStep) Run(ctx context.Context, dry bool) error { - log.Printf("Executing %s %s", s.name, s.config.As) - + if !s.config.SkipLogs { + log.Printf("Executing %s %s", s.name, s.config.As) + } containerResources, err := resourcesFor(s.resources.RequirementsForStep(s.config.As)) if err != nil { return fmt.Errorf("unable to calculate %s pod resources for %s: %s", s.name, s.config.As, err) @@ -107,8 +116,8 @@ func (s *podStep) Run(ctx context.Context, dry bool) error { s.subTests = testCaseNotifier.SubTests(s.Description() + " - ") }() - if err := waitForPodCompletion(s.podClient.Pods(s.jobSpec.Namespace), pod.Name, testCaseNotifier); err != nil { - return fmt.Errorf("test %q failed: %v", pod.Name, err) + if err := waitForPodCompletion(s.podClient.Pods(s.jobSpec.Namespace), pod.Name, testCaseNotifier, s.config.SkipLogs); err != nil { + return fmt.Errorf("%s %q failed: %v", s.name, pod.Name, err) } return nil } @@ -267,3 +276,15 @@ func getSecretVolumeMountFromSecret(secretMountPath string) []coreapi.VolumeMoun }, } } + +// RunPod may be used to run a pod to completion. Provides a simpler interface than +// PodStep and is intended for other steps that may need to run transient actions. +// This pod will not be able to gather artifacts, nor will it report log messages +// unless it fails. +func RunPod(podClient PodClient, pod *coreapi.Pod) error { + pod, err := createOrRestartPod(podClient.Pods(pod.Namespace), pod) + if err != nil { + return err + } + return waitForPodCompletion(podClient.Pods(pod.Namespace), pod.Name, nil, true) +} diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/release/create_release.go b/vendor/github.com/openshift/ci-operator/pkg/steps/release/create_release.go index a72fe69c..29b2d002 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/release/create_release.go +++ b/vendor/github.com/openshift/ci-operator/pkg/steps/release/create_release.go @@ -2,24 +2,53 @@ package release import ( "context" + "encoding/json" "fmt" + "io/ioutil" "log" + "os" + "path/filepath" + "strings" + "time" - imageapi "github.com/openshift/api/image/v1" - imageclientset "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" + coreapi "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" rbacclientset "k8s.io/client-go/kubernetes/typed/rbac/v1" + "k8s.io/client-go/util/retry" + + imageapi "github.com/openshift/api/image/v1" + imageclientset "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" "github.com/openshift/ci-operator/pkg/api" "github.com/openshift/ci-operator/pkg/steps" ) -// assembleReleaseStep knows how to build an update payload image for -// an OpenShift release by waiting for the full release image set to be -// created, then invoking the admin command for building a new release. +// assembleReleaseStep is responsible for creating release images from +// the stable or stable-initial image streams for use with tests that need +// to install or upgrade a cluster. It uses the `cli` image within the +// image stream to create the image and pushes it to a `release` image stream +// at the `latest` or `initial` tags. As output it provides the environment +// variables RELEASE_IMAGE_(LATEST|INITIAL) which can be used by templates +// that invoke the installer. +// +// Since release images describe a set of images, when a user provides +// RELEASE_IMAGE_INITIAL or RELEASE_IMAGE_LATEST as inputs to the ci-operator +// job we treat those as inputs we must expand into the `stable-initial` or +// `stable` image streams. This is because our test scenarios need access not +// just to the release image, but also to the images in that release image +// like installer, cli, or tests. To make it easy for a CI job to install from +// an older release image, we need to extract the 'installer' image into the +// same location that we would expect if it came from a tag_specification. +// The images inside of a release image override any images built or imported +// into the job, which allows you to have an empty tag_specification and +// inject the images from a known historic release for the purposes of building +// branches of those releases. type assembleReleaseStep struct { config api.ReleaseTagConfiguration + latest bool resources api.ResourceConfiguration imageClient imageclientset.ImageV1Interface podClient steps.PodClient @@ -29,21 +58,43 @@ type assembleReleaseStep struct { } func (s *assembleReleaseStep) Inputs(ctx context.Context, dry bool) (api.InputDefinition, error) { + if val := os.Getenv(s.envVar()); len(val) > 0 { + return api.InputDefinition{val}, nil + } return nil, nil } func (s *assembleReleaseStep) Run(ctx context.Context, dry bool) error { - stable, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Get(api.StableImageStream, meta.GetOptions{}) + tag := s.tag() + streamName := s.streamName() + + // if the user specified an input env var, we tag it in instead of generating it + providedImage, ok := os.LookupEnv(s.envVar()) + if ok { + if len(providedImage) == 0 { + log.Printf("No %s release image necessary", tag) + return nil + } + return s.importFromReleaseImage(ctx, dry, providedImage) + } + + stable, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Get(streamName, meta.GetOptions{}) if err != nil { - return fmt.Errorf("could not resolve stable imagestream: %v", err) + if errors.IsNotFound(err) { + // if a user sets IMAGE_FORMAT=... we skip importing the image stream contents, which prevents us from + // generating a release image. + log.Printf("No %s release image can be generated when the %s image stream was skipped", tag, streamName) + return nil + } + return fmt.Errorf("could not resolve imagestream %s: %v", streamName, err) } - cvo, ok := resolvePullSpec(stable, "cluster-version-operator") + cvo, ok := resolvePullSpec(stable, "cluster-version-operator", true) if !ok { - log.Printf("No release image necessary, stable image stream does not include a cluster-version-operator image") + log.Printf("No %s release image necessary, %s image stream does not include a cluster-version-operator image", tag, streamName) return nil } - if _, ok := resolvePullSpec(stable, "cli"); !ok { - return fmt.Errorf("no 'cli' image was tagged into the stable stream, that image is required for building a release") + if _, ok := resolvePullSpec(stable, "cli", true); !ok { + return fmt.Errorf("no 'cli' image was tagged into the %s stream, that image is required for building a release", streamName) } release, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Create(&imageapi.ImageStream{ @@ -61,10 +112,10 @@ func (s *assembleReleaseStep) Run(ctx context.Context, dry bool) error { } } - destination := fmt.Sprintf("%s:%s", release.Status.PublicDockerImageRepository, "latest") - log.Printf("Create a new update payload image %s", destination) + destination := fmt.Sprintf("%s:%s", release.Status.PublicDockerImageRepository, tag) + log.Printf("Create release image %s", destination) podConfig := steps.PodStepConfiguration{ - As: "release-latest", + As: fmt.Sprintf("release-%s", tag), From: api.ImageStreamTagReference{ Name: api.StableImageStream, Tag: "cli", @@ -76,8 +127,8 @@ set -euo pipefail export HOME=/tmp oc registry login oc adm release new --max-per-registry=32 -n %q --from-image-stream %q --to-image-base %q --to-image %q -oc adm release extract --from=%q --to=/tmp/artifacts/release-payload -`, s.jobSpec.Namespace, api.StableImageStream, cvo, destination, destination), +oc adm release extract --from=%q --to=/tmp/artifacts/release-payload-%s +`, s.jobSpec.Namespace, api.StableImageStream, cvo, destination, destination, tag), } // set an explicit default for release-latest resources, but allow customization if necessary @@ -97,22 +148,249 @@ oc adm release extract --from=%q --to=/tmp/artifacts/release-payload return step.Run(ctx, dry) } +// importFromReleaseImage uses the provided release image and updates the stable / release streams as +// appropriate with the contents of the payload so that downstream components are using the correct images. +// The most common case is to use the correct installer image, tests, and cli commands. +func (s *assembleReleaseStep) importFromReleaseImage(ctx context.Context, dry bool, providedImage string) error { + tag := s.tag() + streamName := s.streamName() + + if dry { + return nil + } + + start := time.Now() + + // create the stable image stream with lookup policy + _, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Create(&imageapi.ImageStream{ + ObjectMeta: meta.ObjectMeta{ + Name: streamName, + }, + Spec: imageapi.ImageStreamSpec{ + LookupPolicy: imageapi.ImageLookupPolicy{ + Local: true, + }, + }, + }) + if err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("could not create stable imagestreamtag: %v", err) + } + // tag the release image in and let it import + if _, err := s.imageClient.ImageStreamTags(s.jobSpec.Namespace).Update(&imageapi.ImageStreamTag{ + ObjectMeta: meta.ObjectMeta{ + Name: fmt.Sprintf("release:%s", tag), + }, + Tag: &imageapi.TagReference{ + From: &coreapi.ObjectReference{ + Kind: "DockerImage", + Name: providedImage, + }, + }, + }); err != nil { + return err + } + // wait for the release image to be available + var pullSpec string + if err := wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { + is, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Get("release", meta.GetOptions{}) + if err != nil { + return false, err + } + ref, _ := findStatusTag(is, tag) + if ref == nil { + return false, nil + } + var registry string + if len(is.Status.PublicDockerImageRepository) > 0 { + registry = is.Status.PublicDockerImageRepository + } else if len(is.Status.DockerImageRepository) > 0 { + registry = is.Status.DockerImageRepository + } else { + return false, fmt.Errorf("image stream %s has no accessible image registry value", is.Name) + } + pullSpec = fmt.Sprintf("%s:%s", registry, tag) + return true, nil + }); err != nil { + return fmt.Errorf("unable to import %s release image: %v", tag, err) + } + + // override anything in stable with the contents of the release image + // TODO: should we allow underride for things we built in pipeline? + artifactDir := s.artifactDir + if len(artifactDir) == 0 { + var err error + artifactDir, err = ioutil.TempDir("", "payload-images") + if err != nil { + return fmt.Errorf("unable to create temporary artifact dir for payload extraction") + } + } + + // get the CLI image from the payload (since we need it to run oc adm release extract) + target := fmt.Sprintf("release-images-%s", tag) + targetCLI := fmt.Sprintf("%s-cli", target) + if err := steps.RunPod(s.podClient, &coreapi.Pod{ + ObjectMeta: meta.ObjectMeta{ + Name: targetCLI, + Namespace: s.jobSpec.Namespace, + }, + Spec: coreapi.PodSpec{ + RestartPolicy: coreapi.RestartPolicyNever, + Containers: []coreapi.Container{ + { + Name: "release", + Image: pullSpec, + Command: []string{"/bin/sh", "-c", "cluster-version-operator image cli > /dev/termination-log"}, + }, + }, + }, + }); err != nil { + return fmt.Errorf("unable to find the 'cli' image in the provided release image: %v", err) + } + pod, err := s.podClient.Pods(s.jobSpec.Namespace).Get(targetCLI, meta.GetOptions{}) + if err != nil { + return fmt.Errorf("unable to extract the 'cli' image from the release image: %v", err) + } + if len(pod.Status.ContainerStatuses) == 0 || pod.Status.ContainerStatuses[0].State.Terminated == nil { + return fmt.Errorf("unable to extract the 'cli' image from the release image: %v", err) + } + cliImage := pod.Status.ContainerStatuses[0].State.Terminated.Message + + // tag the cli image into stable so we use the correct pull secrets from the namespace + if _, err := s.imageClient.ImageStreamTags(s.jobSpec.Namespace).Update(&imageapi.ImageStreamTag{ + ObjectMeta: meta.ObjectMeta{ + Name: fmt.Sprintf("%s:cli", streamName), + }, + Tag: &imageapi.TagReference{ + From: &coreapi.ObjectReference{ + Kind: "DockerImage", + Name: cliImage, + }, + }, + }); err != nil { + return err + } + + // run adm release extract and grab the raw image-references from the payload + podConfig := steps.PodStepConfiguration{ + SkipLogs: true, + As: target, + From: api.ImageStreamTagReference{ + Name: api.StableImageStream, + Tag: "cli", + }, + ServiceAccountName: "builder", + ArtifactDir: "/tmp/artifacts", + Commands: fmt.Sprintf(` +set -euo pipefail +export HOME=/tmp +oc registry login +oc adm release extract --from=%q --file=image-references > /tmp/artifacts/%s +`, pullSpec, target), + } + + // set an explicit default for release-latest resources, but allow customization if necessary + resources := s.resources + if _, ok := resources[podConfig.As]; !ok { + copied := make(api.ResourceConfiguration) + for k, v := range resources { + copied[k] = v + } + // max cpu observed at 0.1 core, most memory ~ 420M + copied[podConfig.As] = api.ResourceRequirements{Requests: api.ResourceList{"cpu": "50m", "memory": "400Mi"}} + resources = copied + } + step := steps.PodStep("release", podConfig, resources, s.podClient, artifactDir, s.jobSpec) + if err := step.Run(ctx, false); err != nil { + return err + } + + // read the contents from the artifacts directory + isContents, err := ioutil.ReadFile(filepath.Join(artifactDir, podConfig.As, target)) + if err != nil { + return fmt.Errorf("unable to read release image stream: %v", err) + } + var releaseIS imageapi.ImageStream + if err := json.Unmarshal(isContents, &releaseIS); err != nil { + return fmt.Errorf("unable to decode release image stream: %v", err) + } + if releaseIS.Kind != "ImageStream" || releaseIS.APIVersion != "image.openshift.io/v1" { + return fmt.Errorf("unexpected image stream contents: %v", err) + } + + // update the stable image stream to have all of the tags from the payload + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + stable, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Get(streamName, meta.GetOptions{}) + if err != nil { + return fmt.Errorf("could not resolve imagestream %s: %v", streamName, err) + } + + existing := sets.NewString() + tags := make([]imageapi.TagReference, 0, len(releaseIS.Spec.Tags)+len(stable.Spec.Tags)) + for _, tag := range releaseIS.Spec.Tags { + existing.Insert(tag.Name) + tags = append(tags, tag) + } + for _, tag := range stable.Spec.Tags { + if existing.Has(tag.Name) { + continue + } + existing.Insert(tag.Name) + tags = append(tags, tag) + } + stable.Spec.Tags = tags + + _, err = s.imageClient.ImageStreams(s.jobSpec.Namespace).Update(stable) + return err + }); err != nil { + return fmt.Errorf("unable to update stable image stream with release tags: %v", err) + } + + log.Printf("Imported %s to release:%s and updated %s images in %s", providedImage, tag, streamName, time.Now().Sub(start).Truncate(time.Second)) + return nil +} + func (s *assembleReleaseStep) Done() (bool, error) { // TODO: define done return true, nil } func (s *assembleReleaseStep) Requires() []api.StepLink { - return []api.StepLink{api.ImagesReadyLink()} + // if our prereq is provided, we don't need any prereqs + if len(os.Getenv(s.envVar())) > 0 { + return nil + } + if s.latest { + return []api.StepLink{api.ImagesReadyLink()} + } + return []api.StepLink{api.ReleaseImagesLink()} } func (s *assembleReleaseStep) Creates() []api.StepLink { - return []api.StepLink{api.ReleasePayloadImageLink(api.PipelineImageStreamTagReference("latest"))} + return []api.StepLink{api.ReleasePayloadImageLink(api.PipelineImageStreamTagReference(s.tag()))} +} + +func (s *assembleReleaseStep) tag() string { + if s.latest { + return "latest" + } + return "initial" +} + +func (s *assembleReleaseStep) streamName() string { + if s.latest { + return api.StableImageStream + } + return fmt.Sprintf("%s-initial", api.StableImageStream) +} + +func (s *assembleReleaseStep) envVar() string { + return fmt.Sprintf("RELEASE_IMAGE_%s", strings.ToUpper(s.tag())) } func (s *assembleReleaseStep) Provides() (api.ParameterMap, api.StepLink) { + tag := s.tag() return api.ParameterMap{ - "RELEASE_IMAGE_LATEST": func() (string, error) { + s.envVar(): func() (string, error) { is, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Get("release", meta.GetOptions{}) if err != nil { return "", fmt.Errorf("could not retrieve output imagestream: %v", err) @@ -125,22 +403,28 @@ func (s *assembleReleaseStep) Provides() (api.ParameterMap, api.StepLink) { } else { return "", fmt.Errorf("image stream %s has no accessible image registry value", "release") } - return fmt.Sprintf("%s:%s", registry, "latest"), nil + return fmt.Sprintf("%s:%s", registry, tag), nil }, - }, api.ReleasePayloadImageLink(api.PipelineImageStreamTagReference("latest")) + }, api.ReleasePayloadImageLink(api.PipelineImageStreamTagReference(tag)) } -func (s *assembleReleaseStep) Name() string { return "[release:latest]" } +func (s *assembleReleaseStep) Name() string { + return fmt.Sprintf("[release:%s]", s.tag()) +} func (s *assembleReleaseStep) Description() string { - return fmt.Sprintf("Create a release image in the release image stream") + if s.latest { + return "Create the release image containing all images built by this job" + } + return "Create initial release image from the images that were in the input tag_specification" } // AssembleReleaseStep builds a new update payload image based on the cluster version operator // and the operators defined in the release configuration. -func AssembleReleaseStep(config api.ReleaseTagConfiguration, resources api.ResourceConfiguration, podClient steps.PodClient, imageClient imageclientset.ImageV1Interface, artifactDir string, jobSpec *api.JobSpec) api.Step { +func AssembleReleaseStep(latest bool, config api.ReleaseTagConfiguration, resources api.ResourceConfiguration, podClient steps.PodClient, imageClient imageclientset.ImageV1Interface, artifactDir string, jobSpec *api.JobSpec) api.Step { return &assembleReleaseStep{ config: config, + latest: latest, resources: resources, podClient: podClient, imageClient: imageClient, @@ -148,24 +432,3 @@ func AssembleReleaseStep(config api.ReleaseTagConfiguration, resources api.Resou jobSpec: jobSpec, } } - -func resolvePullSpec(is *imageapi.ImageStream, tag string) (string, bool) { - for _, tags := range is.Status.Tags { - if tags.Tag != tag { - continue - } - if len(tags.Items) == 0 { - break - } - if image := tags.Items[0].Image; len(image) > 0 { - if len(is.Status.PublicDockerImageRepository) > 0 { - return fmt.Sprintf("%s@%s", is.Status.PublicDockerImageRepository, image), true - } - if len(is.Status.DockerImageRepository) > 0 { - return fmt.Sprintf("%s@%s", is.Status.DockerImageRepository, image), true - } - } - break - } - return "", false -} diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/promote.go b/vendor/github.com/openshift/ci-operator/pkg/steps/release/promote.go similarity index 99% rename from vendor/github.com/openshift/ci-operator/pkg/steps/promote.go rename to vendor/github.com/openshift/ci-operator/pkg/steps/release/promote.go index ab62a727..59610994 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/promote.go +++ b/vendor/github.com/openshift/ci-operator/pkg/steps/release/promote.go @@ -1,4 +1,4 @@ -package steps +package release import ( "context" diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/release/release_images.go b/vendor/github.com/openshift/ci-operator/pkg/steps/release/release_images.go new file mode 100644 index 00000000..c5300920 --- /dev/null +++ b/vendor/github.com/openshift/ci-operator/pkg/steps/release/release_images.go @@ -0,0 +1,318 @@ +package release + +import ( + "context" + "encoding/json" + "fmt" + "log" + "strings" + + imageapi "github.com/openshift/api/image/v1" + "github.com/openshift/ci-operator/pkg/api" + imageclientset "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" + routeclientset "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" + coreapi "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + coreclientset "k8s.io/client-go/kubernetes/typed/core/v1" +) + +const ( + ConfigMapName = "release" +) + +// stableImagesTagStep is used when no release configuration is necessary +type stableImagesTagStep struct { + jobSpec *api.JobSpec + dstClient imageclientset.ImageV1Interface +} + +func StableImagesTagStep(dstClient imageclientset.ImageV1Interface, jobSpec *api.JobSpec) api.Step { + return &stableImagesTagStep{ + dstClient: dstClient, + jobSpec: jobSpec, + } +} + +func (s *stableImagesTagStep) Run(ctx context.Context, dry bool) error { + log.Printf("Will output images to %s:%s", api.StableImageStream, api.ComponentFormatReplacement) + + newIS := &imageapi.ImageStream{ + ObjectMeta: meta.ObjectMeta{ + Name: api.StableImageStream, + }, + Spec: imageapi.ImageStreamSpec{ + LookupPolicy: imageapi.ImageLookupPolicy{ + Local: true, + }, + }, + } + if dry { + istJSON, err := json.MarshalIndent(newIS, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal image stream: %v", err) + } + fmt.Printf("%s\n", istJSON) + return nil + } + _, err := s.dstClient.ImageStreams(s.jobSpec.Namespace).Create(newIS) + if err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("could not create stable imagestreamtag: %v", err) + } + return nil +} + +func (s *stableImagesTagStep) Inputs(ctx context.Context, dry bool) (api.InputDefinition, error) { + return nil, nil +} + +func (s *stableImagesTagStep) Done() (bool, error) { return true, nil } + +func (s *stableImagesTagStep) Requires() []api.StepLink { return []api.StepLink{} } + +func (s *stableImagesTagStep) Creates() []api.StepLink { return []api.StepLink{api.ReleaseImagesLink()} } + +func (s *stableImagesTagStep) Provides() (api.ParameterMap, api.StepLink) { return nil, nil } + +func (s *stableImagesTagStep) Name() string { return "[output-images]" } + +func (s *stableImagesTagStep) Description() string { + return fmt.Sprintf("Create the output image stream %s", api.StableImageStream) +} + +// releaseImagesTagStep will tag a full release suite +// of images in from the configured namespace. It is +// expected that builds will overwrite these tags at +// a later point, selectively +type releaseImagesTagStep struct { + config api.ReleaseTagConfiguration + srcClient imageclientset.ImageV1Interface + dstClient imageclientset.ImageV1Interface + routeClient routeclientset.RoutesGetter + configMapClient coreclientset.ConfigMapsGetter + params *api.DeferredParameters + jobSpec *api.JobSpec +} + +func findStatusTag(is *imageapi.ImageStream, tag string) (*coreapi.ObjectReference, string) { + for _, t := range is.Status.Tags { + if t.Tag != tag { + continue + } + if len(t.Items) == 0 { + return nil, "" + } + if len(t.Items[0].Image) == 0 { + return &coreapi.ObjectReference{ + Kind: "DockerImage", + Name: t.Items[0].DockerImageReference, + }, "" + } + return &coreapi.ObjectReference{ + Kind: "ImageStreamImage", + Namespace: is.Namespace, + Name: fmt.Sprintf("%s@%s", is.Name, t.Items[0].Image), + }, t.Items[0].Image + } + return nil, "" +} + +func (s *releaseImagesTagStep) Inputs(ctx context.Context, dry bool) (api.InputDefinition, error) { + return nil, nil +} + +func sourceName(config api.ReleaseTagConfiguration) string { + return fmt.Sprintf("%s/%s:%s", config.Namespace, config.Name, api.ComponentFormatReplacement) +} + +func (s *releaseImagesTagStep) Run(ctx context.Context, dry bool) error { + if dry { + log.Printf("Tagging release images from %s", sourceName(s.config)) + } else { + if format, err := s.imageFormat(); err == nil { + log.Printf("Tagged release images from %s, images will be pullable from %s", sourceName(s.config), format) + } else { + log.Printf("Tagged release images from %s", sourceName(s.config)) + } + } + + is, err := s.srcClient.ImageStreams(s.config.Namespace).Get(s.config.Name, meta.GetOptions{}) + if err != nil { + return fmt.Errorf("could not resolve stable imagestream: %v", err) + } + + // check to see if the src and dst are the same cluster, in which case we can use a more efficient tagging path + if len(s.config.Cluster) > 0 { + if dstIs, err := s.dstClient.ImageStreams(is.Namespace).Get(is.Name, meta.GetOptions{}); err == nil && dstIs.UID == is.UID { + s.config.Cluster = "" + } + } + + var repo string + if len(s.config.Cluster) > 0 { + if len(is.Status.PublicDockerImageRepository) > 0 { + repo = is.Status.PublicDockerImageRepository + } else if len(is.Status.DockerImageRepository) > 0 { + repo = is.Status.DockerImageRepository + } else { + return fmt.Errorf("remote image stream %s has no accessible image registry value", s.config.Name) + } + } + + is.UID = "" + newIS := &imageapi.ImageStream{ + ObjectMeta: meta.ObjectMeta{ + Name: api.StableImageStream, + }, + Spec: imageapi.ImageStreamSpec{ + LookupPolicy: imageapi.ImageLookupPolicy{ + Local: true, + }, + }, + } + for _, tag := range is.Spec.Tags { + if valid, image := findStatusTag(is, tag.Name); valid != nil { + if len(s.config.Cluster) > 0 { + if len(image) > 0 { + valid = &coreapi.ObjectReference{Kind: "DockerImage", Name: fmt.Sprintf("%s@%s", repo, image)} + } else { + valid = &coreapi.ObjectReference{Kind: "DockerImage", Name: fmt.Sprintf("%s:%s", repo, tag.Name)} + } + } + newIS.Spec.Tags = append(newIS.Spec.Tags, imageapi.TagReference{ + Name: tag.Name, + From: valid, + }) + } + } + + if dry { + istJSON, err := json.MarshalIndent(newIS, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal image stream: %v", err) + } + fmt.Printf("%s\n", istJSON) + return nil + } + + initialIS := newIS.DeepCopy() + initialIS.Name = fmt.Sprintf("%s-initial", api.StableImageStream) + + is, err = s.dstClient.ImageStreams(s.jobSpec.Namespace).Create(newIS) + if err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("could not copy stable imagestreamtag: %v", err) + } + + is, err = s.dstClient.ImageStreams(s.jobSpec.Namespace).Create(initialIS) + if err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("could not copy stable-initial imagestreamtag: %v", err) + } + + for _, tag := range is.Spec.Tags { + spec, ok := resolvePullSpec(is, tag.Name, false) + if !ok { + continue + } + s.params.Set(componentToParamName(tag.Name), spec) + } + + return nil +} + +func (s *releaseImagesTagStep) Done() (bool, error) { + return false, nil +} + +func (s *releaseImagesTagStep) Requires() []api.StepLink { + return []api.StepLink{} +} + +func (s *releaseImagesTagStep) Creates() []api.StepLink { + return []api.StepLink{api.ReleaseImagesLink()} +} + +func (s *releaseImagesTagStep) Provides() (api.ParameterMap, api.StepLink) { + return api.ParameterMap{ + "IMAGE_FORMAT": s.imageFormat, + }, api.ImagesReadyLink() +} + +func (s *releaseImagesTagStep) imageFormat() (string, error) { + spec, err := s.repositoryPullSpec() + if err != nil { + return "REGISTRY", err + } + registry := strings.SplitN(spec, "/", 2)[0] + format := fmt.Sprintf("%s/%s/%s:%s", registry, s.jobSpec.Namespace, fmt.Sprintf("%s%s", s.config.NamePrefix, api.StableImageStream), api.ComponentFormatReplacement) + return format, nil +} + +func (s *releaseImagesTagStep) repositoryPullSpec() (string, error) { + is, err := s.dstClient.ImageStreams(s.jobSpec.Namespace).Get(api.PipelineImageStream, meta.GetOptions{}) + if err != nil { + return "", err + } + if len(is.Status.PublicDockerImageRepository) > 0 { + return is.Status.PublicDockerImageRepository, nil + } + if len(is.Status.DockerImageRepository) > 0 { + return is.Status.DockerImageRepository, nil + } + return "", fmt.Errorf("no pull spec available for image stream %s", api.PipelineImageStream) +} + +func (s *releaseImagesTagStep) Name() string { return "[release-inputs]" } + +func (s *releaseImagesTagStep) Description() string { + return fmt.Sprintf("Find all of the input images from %s and tag them into the output image stream", sourceName(s.config)) +} + +func ReleaseImagesTagStep(config api.ReleaseTagConfiguration, srcClient, dstClient imageclientset.ImageV1Interface, routeClient routeclientset.RoutesGetter, configMapClient coreclientset.ConfigMapsGetter, params *api.DeferredParameters, jobSpec *api.JobSpec) api.Step { + // when source and destination client are the same, we don't need to use external imports + if srcClient == dstClient { + config.Cluster = "" + } + return &releaseImagesTagStep{ + config: config, + srcClient: srcClient, + dstClient: dstClient, + routeClient: routeClient, + configMapClient: configMapClient, + params: params, + jobSpec: jobSpec, + } +} + +func componentToParamName(component string) string { + return strings.ToUpper(strings.Replace(component, "-", "_", -1)) +} + +func resolvePullSpec(is *imageapi.ImageStream, tag string, requireExact bool) (string, bool) { + for _, tags := range is.Status.Tags { + if tags.Tag != tag { + continue + } + if len(tags.Items) == 0 { + break + } + if image := tags.Items[0].Image; len(image) > 0 { + if len(is.Status.PublicDockerImageRepository) > 0 { + return fmt.Sprintf("%s@%s", is.Status.PublicDockerImageRepository, image), true + } + if len(is.Status.DockerImageRepository) > 0 { + return fmt.Sprintf("%s@%s", is.Status.DockerImageRepository, image), true + } + } + break + } + if requireExact { + return "", false + } + if len(is.Status.PublicDockerImageRepository) > 0 { + return fmt.Sprintf("%s:%s", is.Status.PublicDockerImageRepository, tag), true + } + if len(is.Status.DockerImageRepository) > 0 { + return fmt.Sprintf("%s:%s", is.Status.DockerImageRepository, tag), true + } + return "", false +} diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/release_images.go b/vendor/github.com/openshift/ci-operator/pkg/steps/release_images.go deleted file mode 100644 index 113352a3..00000000 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/release_images.go +++ /dev/null @@ -1,459 +0,0 @@ -package steps - -import ( - "context" - "encoding/json" - "fmt" - "log" - "strings" - - imageapi "github.com/openshift/api/image/v1" - "github.com/openshift/ci-operator/pkg/api" - imageclientset "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" - routeclientset "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" - coreapi "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - coreclientset "k8s.io/client-go/kubernetes/typed/core/v1" -) - -const ( - ConfigMapName = "release" - - componentFormatReplacement = "${component}" -) - -// stableImagesTagStep is used when no release configuration is necessary -type stableImagesTagStep struct { - jobSpec *api.JobSpec - dstClient imageclientset.ImageV1Interface -} - -func StableImagesTagStep(dstClient imageclientset.ImageV1Interface, jobSpec *api.JobSpec) api.Step { - return &stableImagesTagStep{ - dstClient: dstClient, - jobSpec: jobSpec, - } -} - -func (s *stableImagesTagStep) Run(ctx context.Context, dry bool) error { - log.Printf("Will output images to %s:${component}", api.StableImageStream) - - newIS := &imageapi.ImageStream{ - ObjectMeta: meta.ObjectMeta{ - Name: api.StableImageStream, - }, - Spec: imageapi.ImageStreamSpec{ - LookupPolicy: imageapi.ImageLookupPolicy{ - Local: true, - }, - }, - } - if dry { - istJSON, err := json.MarshalIndent(newIS, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal image stream: %v", err) - } - fmt.Printf("%s\n", istJSON) - return nil - } - _, err := s.dstClient.ImageStreams(s.jobSpec.Namespace).Create(newIS) - if err != nil && !errors.IsAlreadyExists(err) { - return fmt.Errorf("could not create stable imagestreamtag: %v", err) - } - return nil -} - -func (s *stableImagesTagStep) Inputs(ctx context.Context, dry bool) (api.InputDefinition, error) { - return nil, nil -} - -func (s *stableImagesTagStep) Done() (bool, error) { return true, nil } - -func (s *stableImagesTagStep) Requires() []api.StepLink { return []api.StepLink{} } - -func (s *stableImagesTagStep) Creates() []api.StepLink { return []api.StepLink{api.ReleaseImagesLink()} } - -func (s *stableImagesTagStep) Provides() (api.ParameterMap, api.StepLink) { return nil, nil } - -func (s *stableImagesTagStep) Name() string { return "[output-images]" } - -func (s *stableImagesTagStep) Description() string { - return fmt.Sprintf("Create the output image stream %s", api.StableImageStream) -} - -// releaseImagesTagStep will tag a full release suite -// of images in from the configured namespace. It is -// expected that builds will overwrite these tags at -// a later point, selectively -type releaseImagesTagStep struct { - config api.ReleaseTagConfiguration - srcClient imageclientset.ImageV1Interface - dstClient imageclientset.ImageV1Interface - routeClient routeclientset.RoutesGetter - configMapClient coreclientset.ConfigMapsGetter - params *DeferredParameters - jobSpec *api.JobSpec -} - -func findStatusTag(is *imageapi.ImageStream, tag string) (*coreapi.ObjectReference, string) { - for _, t := range is.Status.Tags { - if t.Tag != tag { - continue - } - if len(t.Items) == 0 { - return nil, "" - } - if len(t.Items[0].Image) == 0 { - return &coreapi.ObjectReference{ - Kind: "DockerImage", - Name: t.Items[0].DockerImageReference, - }, "" - } - return &coreapi.ObjectReference{ - Kind: "ImageStreamImage", - Namespace: is.Namespace, - Name: fmt.Sprintf("%s@%s", is.Name, t.Items[0].Image), - }, t.Items[0].Image - } - return nil, "" -} - -func (s *releaseImagesTagStep) Inputs(ctx context.Context, dry bool) (api.InputDefinition, error) { - return nil, nil -} - -func sourceName(config api.ReleaseTagConfiguration) string { - if len(config.Name) > 0 { - return fmt.Sprintf("%s/%s:${component}", config.Namespace, config.Name) - } - return fmt.Sprintf("%s/${component}:%s", config.Namespace, config.Tag) -} - -func (s *releaseImagesTagStep) Run(ctx context.Context, dry bool) error { - if dry { - log.Printf("Tagging release images from %s", sourceName(s.config)) - } else { - if format, err := s.imageFormat(); err == nil { - log.Printf("Tagged release images from %s, images will be pullable from %s", sourceName(s.config), format) - } else { - log.Printf("Tagged release images from %s", sourceName(s.config)) - } - } - - if len(s.config.Name) > 0 { - is, err := s.srcClient.ImageStreams(s.config.Namespace).Get(s.config.Name, meta.GetOptions{}) - if err != nil { - return fmt.Errorf("could not resolve stable imagestream: %v", err) - } - - // check to see if the src and dst are the same cluster, in which case we can use a more efficient tagging path - if len(s.config.Cluster) > 0 { - if dstIs, err := s.dstClient.ImageStreams(is.Namespace).Get(is.Name, meta.GetOptions{}); err == nil && dstIs.UID == is.UID { - s.config.Cluster = "" - } - } - - var repo string - if len(s.config.Cluster) > 0 { - if len(is.Status.PublicDockerImageRepository) > 0 { - repo = is.Status.PublicDockerImageRepository - } else if len(is.Status.DockerImageRepository) > 0 { - repo = is.Status.DockerImageRepository - } else { - return fmt.Errorf("remote image stream %s has no accessible image registry value", s.config.Name) - } - } - - is.UID = "" - newIS := &imageapi.ImageStream{ - ObjectMeta: meta.ObjectMeta{ - Name: api.StableImageStream, - }, - Spec: imageapi.ImageStreamSpec{ - LookupPolicy: imageapi.ImageLookupPolicy{ - Local: true, - }, - }, - } - for _, tag := range is.Spec.Tags { - if valid, image := findStatusTag(is, tag.Name); valid != nil { - if len(s.config.Cluster) > 0 { - if len(image) > 0 { - valid = &coreapi.ObjectReference{Kind: "DockerImage", Name: fmt.Sprintf("%s@%s", repo, image)} - } else { - valid = &coreapi.ObjectReference{Kind: "DockerImage", Name: fmt.Sprintf("%s:%s", repo, tag.Name)} - } - } - newIS.Spec.Tags = append(newIS.Spec.Tags, imageapi.TagReference{ - Name: tag.Name, - From: valid, - }) - } - } - - if dry { - istJSON, err := json.MarshalIndent(newIS, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal image stream: %v", err) - } - fmt.Printf("%s\n", istJSON) - return nil - } - is, err = s.dstClient.ImageStreams(s.jobSpec.Namespace).Create(newIS) - if err != nil && !errors.IsAlreadyExists(err) { - return fmt.Errorf("could not copy stable imagestreamtag: %v", err) - } - - for _, tag := range is.Spec.Tags { - spec, ok := resolvePullSpec(is, tag.Name) - if !ok { - continue - } - s.params.Set(componentToParamName(tag.Name), spec) - } - - return nil - } - - stableImageStreams, err := s.srcClient.ImageStreams(s.config.Namespace).List(meta.ListOptions{}) - if err != nil { - return fmt.Errorf("could not resolve stable imagestreams: %v", err) - } - - for i, stableImageStream := range stableImageStreams.Items { - log.Printf("Considering stable image stream %s", stableImageStream.Name) - targetTag := s.config.Tag - if override, ok := s.config.TagOverrides[stableImageStream.Name]; ok { - targetTag = override - } - - // check exactly once to see if the src and dst are the same cluster, in which case we can use a more efficient tagging path - if i == 0 && len(s.config.Cluster) > 0 { - if dstIs, err := s.dstClient.ImageStreams(stableImageStream.Namespace).Get(stableImageStream.Name, meta.GetOptions{}); err == nil && dstIs.UID == stableImageStream.UID { - s.config.Cluster = "" - } - } - - var repo string - if len(s.config.Cluster) > 0 { - if len(stableImageStream.Status.PublicDockerImageRepository) > 0 { - repo = stableImageStream.Status.PublicDockerImageRepository - } else if len(stableImageStream.Status.DockerImageRepository) > 0 { - repo = stableImageStream.Status.DockerImageRepository - } else { - return fmt.Errorf("remote image stream %s has no accessible image registry value", s.config.Name) - } - } - - for _, tag := range stableImageStream.Spec.Tags { - if tag.Name == targetTag { - log.Printf("Cross-tagging %s:%s from %s/%s:%s", stableImageStream.Name, targetTag, stableImageStream.Namespace, stableImageStream.Name, targetTag) - var id string - for _, tagStatus := range stableImageStream.Status.Tags { - if tagStatus.Tag == targetTag { - id = tagStatus.Items[0].Image - } - } - if len(id) == 0 { - return fmt.Errorf("no image found backing %s/%s:%s", stableImageStream.Namespace, stableImageStream.Name, targetTag) - } - ist := &imageapi.ImageStreamTag{ - ObjectMeta: meta.ObjectMeta{ - Namespace: s.jobSpec.Namespace, - Name: fmt.Sprintf("%s:%s", stableImageStream.Name, targetTag), - }, - Tag: &imageapi.TagReference{ - Name: targetTag, - From: &coreapi.ObjectReference{ - Kind: "ImageStreamImage", - Name: fmt.Sprintf("%s@%s", stableImageStream.Name, id), - Namespace: s.config.Namespace, - }, - }, - } - - if len(s.config.Cluster) > 0 { - ist.Tag.From = &coreapi.ObjectReference{Kind: "DockerImage", Name: fmt.Sprintf("%s@%s", repo, id)} - } - - if dry { - istJSON, err := json.MarshalIndent(ist, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal imagestreamtag: %v", err) - } - fmt.Printf("%s\n", istJSON) - continue - } - ist, err := s.dstClient.ImageStreamTags(s.jobSpec.Namespace).Create(ist) - if err != nil && !errors.IsAlreadyExists(err) { - return fmt.Errorf("could not copy stable imagestreamtag: %v", err) - } - - if spec, ok := resolvePullSpec(&stableImageStream, tag.Name); ok { - s.params.Set(componentToParamName(tag.Name), spec) - } - } - } - } - - return nil -} - -func (s *releaseImagesTagStep) createReleaseConfigMap(dry bool) error { - imageBase := "dry-fake" - rpmRepo := "dry-fake" - if !dry { - originImageStream, err := s.dstClient.ImageStreams(s.jobSpec.Namespace).Get("origin", meta.GetOptions{}) - if err != nil { - return fmt.Errorf("could not resolve main release ImageStream: %v", err) - } - if len(originImageStream.Status.PublicDockerImageRepository) == 0 { - return fmt.Errorf("release ImageStream %s/%s is not exposed externally", originImageStream.Namespace, originImageStream.Name) - } - imageBase = originImageStream.Status.PublicDockerImageRepository - - rpmRepoServer, err := s.routeClient.Routes(s.jobSpec.Namespace).Get(RPMRepoName, meta.GetOptions{}) - if !errors.IsNotFound(err) { - return fmt.Errorf("could not retrieve RPM repo server route: %v", err) - } else { - rpmRepoServer, err = s.routeClient.Routes(s.config.Namespace).Get(RPMRepoName, meta.GetOptions{}) - if err != nil { - return fmt.Errorf("could not retrieve RPM repo server route: %v", err) - } - } - rpmRepo = rpmRepoServer.Spec.Host - } - - cm := &coreapi.ConfigMap{ - ObjectMeta: meta.ObjectMeta{ - Name: ConfigMapName, - Namespace: s.jobSpec.Namespace, - }, - Data: map[string]string{ - "image-base": imageBase, - "rpm-repo": rpmRepo, - }, - } - if dry { - cmJSON, err := json.MarshalIndent(cm, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal configmap: %v", err) - } - fmt.Printf("%s\n", cmJSON) - return nil - } - if _, err := s.configMapClient.ConfigMaps(s.jobSpec.Namespace).Create(cm); err != nil && !errors.IsAlreadyExists(err) { - return fmt.Errorf("could not create release configmap: %v", err) - } - return nil -} - -func (s *releaseImagesTagStep) Done() (bool, error) { - log.Printf("Checking for existence of %s ConfigMap", ConfigMapName) - if _, err := s.configMapClient.ConfigMaps(s.jobSpec.Namespace).Get(ConfigMapName, meta.GetOptions{}); err != nil { - if errors.IsNotFound(err) { - return false, nil - } else { - return false, fmt.Errorf("could not retrieve release configmap: %v", err) - } - } else { - return true, nil - } -} - -func (s *releaseImagesTagStep) Requires() []api.StepLink { - return []api.StepLink{} -} - -func (s *releaseImagesTagStep) Creates() []api.StepLink { - return []api.StepLink{api.ReleaseImagesLink()} -} - -func (s *releaseImagesTagStep) Provides() (api.ParameterMap, api.StepLink) { - return api.ParameterMap{ - "IMAGE_FORMAT": s.imageFormat, - }, api.ImagesReadyLink() -} - -func (s *releaseImagesTagStep) imageFormat() (string, error) { - spec, err := s.repositoryPullSpec() - if err != nil { - return "REGISTRY", err - } - registry := strings.SplitN(spec, "/", 2)[0] - var format string - if len(s.config.Name) > 0 { - format = fmt.Sprintf("%s/%s/%s:%s", registry, s.jobSpec.Namespace, fmt.Sprintf("%s%s", s.config.NamePrefix, api.StableImageStream), componentFormatReplacement) - } else { - format = fmt.Sprintf("%s/%s/%s:%s", registry, s.jobSpec.Namespace, fmt.Sprintf("%s%s", s.config.NamePrefix, componentFormatReplacement), s.config.Tag) - } - return format, nil -} - -func (s *releaseImagesTagStep) repositoryPullSpec() (string, error) { - is, err := s.dstClient.ImageStreams(s.jobSpec.Namespace).Get(api.PipelineImageStream, meta.GetOptions{}) - if err != nil { - return "", err - } - if len(is.Status.PublicDockerImageRepository) > 0 { - return is.Status.PublicDockerImageRepository, nil - } - if len(is.Status.DockerImageRepository) > 0 { - return is.Status.DockerImageRepository, nil - } - return "", fmt.Errorf("no pull spec available for image stream %s", api.PipelineImageStream) -} - -func (s *releaseImagesTagStep) Name() string { return "[release-inputs]" } - -func (s *releaseImagesTagStep) Description() string { - return fmt.Sprintf("Find all of the input images from %s and tag them into the output image stream", sourceName(s.config)) -} - -func ReleaseImagesTagStep(config api.ReleaseTagConfiguration, srcClient, dstClient imageclientset.ImageV1Interface, routeClient routeclientset.RoutesGetter, configMapClient coreclientset.ConfigMapsGetter, params *DeferredParameters, jobSpec *api.JobSpec) api.Step { - // when source and destination client are the same, we don't need to use external imports - if srcClient == dstClient { - config.Cluster = "" - } - return &releaseImagesTagStep{ - config: config, - srcClient: srcClient, - dstClient: dstClient, - routeClient: routeClient, - configMapClient: configMapClient, - params: params, - jobSpec: jobSpec, - } -} - -func componentToParamName(component string) string { - return strings.ToUpper(strings.Replace(component, "-", "_", -1)) -} - -func resolvePullSpec(is *imageapi.ImageStream, tag string) (string, bool) { - for _, tags := range is.Status.Tags { - if tags.Tag != tag { - continue - } - if len(tags.Items) == 0 { - break - } - if image := tags.Items[0].Image; len(image) > 0 { - if len(is.Status.PublicDockerImageRepository) > 0 { - return fmt.Sprintf("%s@%s", is.Status.PublicDockerImageRepository, image), true - } - if len(is.Status.DockerImageRepository) > 0 { - return fmt.Sprintf("%s@%s", is.Status.DockerImageRepository, image), true - } - } - break - } - if len(is.Status.PublicDockerImageRepository) > 0 { - return fmt.Sprintf("%s:%s", is.Status.PublicDockerImageRepository, tag), true - } - if len(is.Status.DockerImageRepository) > 0 { - return fmt.Sprintf("%s:%s", is.Status.DockerImageRepository, tag), true - } - return "", false -} diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/rpm_injection.go b/vendor/github.com/openshift/ci-operator/pkg/steps/rpm_injection.go index f91cd220..27e6ccbd 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/rpm_injection.go +++ b/vendor/github.com/openshift/ci-operator/pkg/steps/rpm_injection.go @@ -14,7 +14,7 @@ import ( func rpmInjectionDockerfile(from api.PipelineImageStreamTagReference, repo string) string { return fmt.Sprintf(`FROM %s:%s -RUN echo $'[built]\nname = Built RPMs\nbaseurl = http://%s\ngpgcheck = 0\nenabled = 0\n\n[origin-local-release]\nname = Built RPMs\nbaseurl = http://%s\ngpgcheck = 0\nenabled = 0' > /etc/yum.repos.d/built.repo`, api.PipelineImageStream, from, repo, repo) +RUN echo $'[built]\nname = Built RPMs\nbaseurl = http://%s/\ngpgcheck = 0\nenabled = 0\n\n[origin-local-release]\nname = Built RPMs\nbaseurl = http://%s/\ngpgcheck = 0\nenabled = 0' > /etc/yum.repos.d/built.repo`, api.PipelineImageStream, from, repo, repo) } type rpmImageInjectionStep struct { diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/source.go b/vendor/github.com/openshift/ci-operator/pkg/steps/source.go index 8ec7fbd1..94b0b250 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/source.go +++ b/vendor/github.com/openshift/ci-operator/pkg/steps/source.go @@ -42,9 +42,12 @@ var ( ) func sourceDockerfile(fromTag api.PipelineImageStreamTagReference, pathAlias string, job *api.JobSpec) string { - workingDir := fmt.Sprintf("github.com/%s/%s", job.Refs.Org, job.Refs.Repo) - if len(pathAlias) > 0 { - workingDir = pathAlias + workingDir := pathAlias + if len(workingDir) == 0 && job.Refs != nil { + workingDir = fmt.Sprintf("github.com/%s/%s", job.Refs.Org, job.Refs.Repo) + } + if len(workingDir) == 0 && len(job.ExtraRefs) > 0 { + workingDir = fmt.Sprintf("github.com/%s/%s", job.ExtraRefs[0].Org, job.ExtraRefs[0].Repo) } return fmt.Sprintf(` FROM %s:%s diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/template.go b/vendor/github.com/openshift/ci-operator/pkg/steps/template.go index d3569f72..83438782 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/template.go +++ b/vendor/github.com/openshift/ci-operator/pkg/steps/template.go @@ -10,7 +10,6 @@ import ( "path/filepath" "sort" "strings" - "sync" "time" "k8s.io/client-go/rest" @@ -31,7 +30,7 @@ import ( type templateExecutionStep struct { template *templateapi.Template - params *DeferredParameters + params *api.DeferredParameters templateClient TemplateClient podClient PodClient artifactDir string @@ -77,7 +76,7 @@ func (s *templateExecutionStep) Run(ctx context.Context, dry bool) error { if err != nil { return fmt.Errorf("could not resolve image format: %v", err) } - s.template.Parameters[i].Value = strings.Replace(format, componentFormatReplacement, component, -1) + s.template.Parameters[i].Value = strings.Replace(format, api.ComponentFormatReplacement, component, -1) } } } @@ -157,7 +156,7 @@ func (s *templateExecutionStep) Run(ctx context.Context, dry bool) error { for _, ref := range instance.Status.Objects { switch { case ref.Ref.Kind == "Pod" && ref.Ref.APIVersion == "v1": - err := waitForPodCompletion(s.podClient.Pods(s.jobSpec.Namespace), ref.Ref.Name, testCaseNotifier) + err := waitForPodCompletion(s.podClient.Pods(s.jobSpec.Namespace), ref.Ref.Name, testCaseNotifier, false) s.subTests = append(s.subTests, testCaseNotifier.SubTests(fmt.Sprintf("%s - %s ", s.Description(), ref.Ref.Name))...) if err != nil { return fmt.Errorf("template pod %q failed: %v", ref.Ref.Name, err) @@ -230,7 +229,7 @@ func (s *templateExecutionStep) Description() string { return fmt.Sprintf("Run template %s", s.template.Name) } -func TemplateExecutionStep(template *templateapi.Template, params *DeferredParameters, podClient PodClient, templateClient TemplateClient, artifactDir string, jobSpec *api.JobSpec) api.Step { +func TemplateExecutionStep(template *templateapi.Template, params *api.DeferredParameters, podClient PodClient, templateClient TemplateClient, artifactDir string, jobSpec *api.JobSpec) api.Step { return &templateExecutionStep{ template: template, params: params, @@ -241,115 +240,6 @@ func TemplateExecutionStep(template *templateapi.Template, params *DeferredParam } } -type DeferredParameters struct { - lock sync.Mutex - fns api.ParameterMap - values map[string]string - links map[string][]api.StepLink -} - -func NewDeferredParameters() *DeferredParameters { - return &DeferredParameters{ - fns: make(api.ParameterMap), - values: make(map[string]string), - links: make(map[string][]api.StepLink), - } -} - -func (p *DeferredParameters) Map() (map[string]string, error) { - p.lock.Lock() - defer p.lock.Unlock() - m := make(map[string]string) - for k, fn := range p.fns { - if v, ok := p.values[k]; ok { - m[k] = v - continue - } - v, err := fn() - if err != nil { - return nil, fmt.Errorf("could not lazily evaluate deferred parameter: %v", err) - } - p.values[k] = v - m[k] = v - } - return m, nil -} - -func (p *DeferredParameters) Set(name, value string) { - p.lock.Lock() - defer p.lock.Unlock() - if _, ok := p.fns[name]; ok { - return - } - if _, ok := p.values[name]; ok { - return - } - p.values[name] = value -} - -func (p *DeferredParameters) Add(name string, link api.StepLink, fn func() (string, error)) { - p.lock.Lock() - defer p.lock.Unlock() - p.fns[name] = fn - if link != nil { - p.links[name] = []api.StepLink{link} - } -} - -func (p *DeferredParameters) Has(name string) bool { - p.lock.Lock() - defer p.lock.Unlock() - _, ok := p.fns[name] - if ok { - return true - } - _, ok = os.LookupEnv(name) - return ok -} - -func (p *DeferredParameters) Links(name string) []api.StepLink { - p.lock.Lock() - defer p.lock.Unlock() - if _, ok := os.LookupEnv(name); ok { - return nil - } - return p.links[name] -} - -func (p *DeferredParameters) AllLinks() []api.StepLink { - p.lock.Lock() - defer p.lock.Unlock() - var links []api.StepLink - for name, v := range p.links { - if _, ok := os.LookupEnv(name); ok { - continue - } - links = append(links, v...) - } - return links -} - -func (p *DeferredParameters) Get(name string) (string, error) { - p.lock.Lock() - defer p.lock.Unlock() - if value, ok := p.values[name]; ok { - return value, nil - } - if value, ok := os.LookupEnv(name); ok { - p.values[name] = value - return value, nil - } - if fn, ok := p.fns[name]; ok { - value, err := fn() - if err != nil { - return "", fmt.Errorf("could not lazily evaluate deferred parameter: %v", err) - } - p.values[name] = value - return value, nil - } - return "", nil -} - type TemplateClient interface { templateclientset.TemplateV1Interface Process(namespace string, template *templateapi.Template) (*templateapi.Template, error) @@ -556,11 +446,10 @@ func waitForCompletedPodDeletion(podClient coreclientset.PodInterface, name stri return waitForPodDeletion(podClient, name, uid) } -func waitForPodCompletion(podClient coreclientset.PodInterface, name string, notifier ContainerNotifier) error { +func waitForPodCompletion(podClient coreclientset.PodInterface, name string, notifier ContainerNotifier, skipLogs bool) error { if notifier == nil { notifier = NopNotifier } - skipLogs := false completed := make(map[string]time.Time) for { retry, err := waitForPodCompletionOrTimeout(podClient, name, completed, notifier, skipLogs) @@ -604,7 +493,7 @@ func waitForPodCompletionOrTimeout(podClient coreclientset.PodInterface, name st if pod.Spec.RestartPolicy == coreapi.RestartPolicyAlways { return false, nil } - podLogNewFailedContainers(podClient, pod, completed, notifier) + podLogNewFailedContainers(podClient, pod, completed, notifier, skipLogs) if podJobIsOK(pod) { if !skipLogs { log.Printf("Pod %s already succeeded in %s", pod.Name, podDuration(pod).Truncate(time.Second)) @@ -622,9 +511,11 @@ func waitForPodCompletionOrTimeout(podClient coreclientset.PodInterface, name st return true, nil } if pod, ok := event.Object.(*coreapi.Pod); ok { - podLogNewFailedContainers(podClient, pod, completed, notifier) + podLogNewFailedContainers(podClient, pod, completed, notifier, skipLogs) if podJobIsOK(pod) { - log.Printf("Pod %s succeeded after %s", pod.Name, podDuration(pod).Truncate(time.Second)) + if !skipLogs { + log.Printf("Pod %s succeeded after %s", pod.Name, podDuration(pod).Truncate(time.Second)) + } return false, nil } if podJobIsFailed(pod) { @@ -633,7 +524,7 @@ func waitForPodCompletionOrTimeout(podClient coreclientset.PodInterface, name st continue } if event.Type == watch.Deleted { - podLogNewFailedContainers(podClient, pod, completed, notifier) + podLogNewFailedContainers(podClient, pod, completed, notifier, skipLogs) return false, appendLogToError(fmt.Errorf("the pod %s/%s was deleted without completing after %s (failed containers: %s)", pod.Namespace, pod.Name, podDuration(pod).Truncate(time.Second), strings.Join(failedContainerNames(pod), ", ")), podMessages(pod)) } log.Printf("error: Unrecognized event in watch: %v %#v", event.Type, event.Object) @@ -790,7 +681,7 @@ func failedContainerNames(pod *coreapi.Pod) []string { return names } -func podLogNewFailedContainers(podClient coreclientset.PodInterface, pod *coreapi.Pod, completed map[string]time.Time, notifier ContainerNotifier) { +func podLogNewFailedContainers(podClient coreclientset.PodInterface, pod *coreapi.Pod, completed map[string]time.Time, notifier ContainerNotifier, skipLogs bool) { var statuses []coreapi.ContainerStatus statuses = append(statuses, pod.Status.InitContainerStatuses...) statuses = append(statuses, pod.Status.ContainerStatuses...) @@ -807,7 +698,9 @@ func podLogNewFailedContainers(podClient coreclientset.PodInterface, pod *coreap notifier.Notify(pod, status.Name) if s.ExitCode == 0 { - log.Printf("Container %s in pod %s completed successfully", status.Name, pod.Name) + if !skipLogs { + log.Printf("Container %s in pod %s completed successfully", status.Name, pod.Name) + } continue } diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/write_params.go b/vendor/github.com/openshift/ci-operator/pkg/steps/write_params.go index 0f3abb15..72d12614 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/write_params.go +++ b/vendor/github.com/openshift/ci-operator/pkg/steps/write_params.go @@ -13,7 +13,7 @@ import ( ) type writeParametersStep struct { - params *DeferredParameters + params *api.DeferredParameters paramFile string } @@ -69,7 +69,7 @@ func (s *writeParametersStep) Name() string { return "parameters/write" } func (s *writeParametersStep) Description() string { return "Write the job parameters to disk" } -func WriteParametersStep(params *DeferredParameters, paramFile string) api.Step { +func WriteParametersStep(params *api.DeferredParameters, paramFile string) api.Step { return &writeParametersStep{ params: params, paramFile: paramFile, diff --git a/vendor/github.com/openshift/ci-operator/pkg/steps/write_params_test.go b/vendor/github.com/openshift/ci-operator/pkg/steps/write_params_test.go index d040dd9b..c80e358c 100644 --- a/vendor/github.com/openshift/ci-operator/pkg/steps/write_params_test.go +++ b/vendor/github.com/openshift/ci-operator/pkg/steps/write_params_test.go @@ -10,7 +10,7 @@ import ( ) func TestWriteParamsStep(t *testing.T) { - params := NewDeferredParameters() + params := api.NewDeferredParameters() params.Add("K1", someStepLink("another-step"), func() (string, error) { return "V1", nil }) params.Add("K2", someStepLink("another-step"), func() (string, error) { return "V:2", nil }) paramFile, err := ioutil.TempFile("", "") From 121bcebdd0848bfff76ebc6a4cd13fb1ed10d5b5 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Thu, 7 Mar 2019 00:13:21 -0500 Subject: [PATCH 2/3] Update to match removal of release tag configuration `tag` field --- pkg/diffs/diffs_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/diffs/diffs_test.go b/pkg/diffs/diffs_test.go index 5d1fb23f..02dbfb0d 100644 --- a/pkg/diffs/diffs_test.go +++ b/pkg/diffs/diffs_test.go @@ -7,7 +7,7 @@ import ( "github.com/getlantern/deepcopy" "github.com/sirupsen/logrus" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,7 +29,7 @@ func TestGetChangedCiopConfigs(t *testing.T) { ReleaseTagConfiguration: &cioperatorapi.ReleaseTagConfiguration{ Cluster: "kluster", Namespace: "namespace", - Tag: "tag", + Name: "name", }, }, } @@ -65,14 +65,14 @@ func TestGetChangedCiopConfigs(t *testing.T) { before := config.CompoundCiopConfig{"org-repo-branch.yaml": &baseCiopConfig} afterConfig := cioperatorapi.ReleaseBuildConfiguration{} deepcopy.Copy(&afterConfig, baseCiopConfig) - afterConfig.InputConfiguration.ReleaseTagConfiguration.Tag = "another-tag" + afterConfig.InputConfiguration.ReleaseTagConfiguration.Name = "another-name" after := config.CompoundCiopConfig{"org-repo-branch.yaml": &afterConfig} return before, after }, expected: func() config.CompoundCiopConfig { expected := cioperatorapi.ReleaseBuildConfiguration{} deepcopy.Copy(&expected, baseCiopConfig) - expected.InputConfiguration.ReleaseTagConfiguration.Tag = "another-tag" + expected.InputConfiguration.ReleaseTagConfiguration.Name = "another-name" return config.CompoundCiopConfig{"org-repo-branch.yaml": &expected} }, }} From ee595cf24846087314d08a93d129a809eaa227c2 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Thu, 7 Mar 2019 00:26:06 -0500 Subject: [PATCH 3/3] The new upgrade job is supported as part of the ci-operator Instead of mounting the template, the ci-operator can run the job itself. Use the upgrade=true flag only to test this logic while we verify that upgrade tests (and their attendant complexity) don't destabilize other jobs. --- cmd/ci-operator-prowgen/main.go | 40 ++++++++++--------- .../expected_jobs.yaml | 4 -- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/cmd/ci-operator-prowgen/main.go b/cmd/ci-operator-prowgen/main.go index 9f7451ac..e7795f8b 100644 --- a/cmd/ci-operator-prowgen/main.go +++ b/cmd/ci-operator-prowgen/main.go @@ -138,7 +138,9 @@ func generatePodSpecTemplate(info *config.Info, release string, test *cioperator clusterProfile = conf.ClusterProfile needsReleaseRpms = true } else if conf := test.OpenshiftInstallerClusterTestConfiguration; conf != nil { - template = "cluster-launch-installer-e2e" + if !conf.Upgrade { + template = "cluster-launch-installer-e2e" + } clusterProfile = conf.ClusterProfile } else if conf := test.OpenshiftInstallerSrcClusterTestConfiguration; conf != nil { template = "cluster-launch-installer-src" @@ -185,8 +187,8 @@ func generatePodSpecTemplate(info *config.Info, release string, test *cioperator }, }) } - podSpec.Volumes = []kubeapi.Volume{ - { + if len(template) > 0 { + podSpec.Volumes = append(podSpec.Volumes, kubeapi.Volume{ Name: "job-definition", VolumeSource: kubeapi.VolumeSource{ ConfigMap: &kubeapi.ConfigMapVolumeSource{ @@ -195,25 +197,25 @@ func generatePodSpecTemplate(info *config.Info, release string, test *cioperator }, }, }, - }, - clusterProfileVolume, + }) } + podSpec.Volumes = append(podSpec.Volumes, clusterProfileVolume) container := &podSpec.Containers[0] - container.Args = append( - container.Args, - fmt.Sprintf("--secret-dir=%s", clusterProfilePath), - fmt.Sprintf("--template=%s", templatePath)) - container.VolumeMounts = []kubeapi.VolumeMount{ - {Name: "cluster-profile", MountPath: clusterProfilePath}, - {Name: "job-definition", MountPath: templatePath, SubPath: fmt.Sprintf("%s.yaml", template)}, - } - container.Env = append( - container.Env, - kubeapi.EnvVar{Name: "CLUSTER_TYPE", Value: targetCloud}, - kubeapi.EnvVar{Name: "JOB_NAME_SAFE", Value: strings.Replace(test.As, "_", "-", -1)}, - kubeapi.EnvVar{Name: "TEST_COMMAND", Value: test.Commands}) + container.Args = append(container.Args, fmt.Sprintf("--secret-dir=%s", clusterProfilePath)) + if len(template) > 0 { + container.Args = append(container.Args, fmt.Sprintf("--template=%s", templatePath)) + } + container.VolumeMounts = append(container.VolumeMounts, kubeapi.VolumeMount{Name: "cluster-profile", MountPath: clusterProfilePath}) + if len(template) > 0 { + container.VolumeMounts = append(container.VolumeMounts, kubeapi.VolumeMount{Name: "job-definition", MountPath: templatePath, SubPath: fmt.Sprintf("%s.yaml", template)}) + container.Env = append( + container.Env, + kubeapi.EnvVar{Name: "CLUSTER_TYPE", Value: targetCloud}, + kubeapi.EnvVar{Name: "JOB_NAME_SAFE", Value: strings.Replace(test.As, "_", "-", -1)}, + kubeapi.EnvVar{Name: "TEST_COMMAND", Value: test.Commands}) + } if needsReleaseRpms && (info.Org != "openshift" || info.Repo != "origin") { - var repoPath string = fmt.Sprintf("https://rpms.svc.ci.openshift.org/openshift-origin-v%s/", release) + var repoPath = fmt.Sprintf("https://rpms.svc.ci.openshift.org/openshift-origin-v%s/", release) if strings.HasPrefix(release, "origin-v") { repoPath = fmt.Sprintf("https://rpms.svc.ci.openshift.org/openshift-%s/", release) } diff --git a/test/pj-rehearse-integration/expected_jobs.yaml b/test/pj-rehearse-integration/expected_jobs.yaml index 3f3ff5a6..afbeab51 100644 --- a/test/pj-rehearse-integration/expected_jobs.yaml +++ b/test/pj-rehearse-integration/expected_jobs.yaml @@ -59,7 +59,6 @@ name: release namespace: openshift tag: golang-1.10 - canonical_go_repository: "" images: - from: base to: test-image @@ -158,7 +157,6 @@ name: release namespace: openshift tag: golang-1.10 - canonical_go_repository: "" images: - from: base to: test-image @@ -257,7 +255,6 @@ name: release namespace: openshift tag: golang-1.10 - canonical_go_repository: "" images: - from: base to: test-image @@ -357,7 +354,6 @@ name: release namespace: openshift tag: golang-1.10 - canonical_go_repository: "" images: - from: base to: test-image