diff --git a/pkg/api/parameters.go b/pkg/api/parameters.go new file mode 100644 index 00000000..bdb06d1b --- /dev/null +++ b/pkg/api/parameters.go @@ -0,0 +1,116 @@ +package api + +import ( + "fmt" + "os" + "sync" +) + +type DeferredParameters struct { + lock sync.Mutex + fns ParameterMap + values map[string]string + links map[string][]StepLink +} + +func NewDeferredParameters() *DeferredParameters { + return &DeferredParameters{ + fns: make(ParameterMap), + values: make(map[string]string), + links: make(map[string][]StepLink), + } +} + +func (p *DeferredParameters) Map() (map[string]string, error) { + p.lock.Lock() + defer p.lock.Unlock() + m := make(map[string]string) + for k, fn := range p.fns { + if v, ok := p.values[k]; ok { + m[k] = v + continue + } + v, err := fn() + if err != nil { + return nil, fmt.Errorf("could not lazily evaluate deferred parameter: %v", err) + } + p.values[k] = v + m[k] = v + } + return m, nil +} + +func (p *DeferredParameters) Set(name, value string) { + p.lock.Lock() + defer p.lock.Unlock() + if _, ok := p.fns[name]; ok { + return + } + if _, ok := p.values[name]; ok { + return + } + p.values[name] = value +} + +func (p *DeferredParameters) Add(name string, link StepLink, fn func() (string, error)) { + p.lock.Lock() + defer p.lock.Unlock() + p.fns[name] = fn + if link != nil { + p.links[name] = []StepLink{link} + } +} + +func (p *DeferredParameters) Has(name string) bool { + p.lock.Lock() + defer p.lock.Unlock() + _, ok := p.fns[name] + if ok { + return true + } + _, ok = os.LookupEnv(name) + return ok +} + +func (p *DeferredParameters) Links(name string) []StepLink { + p.lock.Lock() + defer p.lock.Unlock() + if _, ok := os.LookupEnv(name); ok { + return nil + } + return p.links[name] +} + +func (p *DeferredParameters) AllLinks() []StepLink { + p.lock.Lock() + defer p.lock.Unlock() + var links []StepLink + for name, v := range p.links { + if _, ok := os.LookupEnv(name); ok { + continue + } + links = append(links, v...) + } + return links +} + +func (p *DeferredParameters) Get(name string) (string, error) { + p.lock.Lock() + defer p.lock.Unlock() + if value, ok := p.values[name]; ok { + return value, nil + } + if value, ok := os.LookupEnv(name); ok { + p.values[name] = value + return value, nil + } + if fn, ok := p.fns[name]; ok { + value, err := fn() + if err != nil { + return "", fmt.Errorf("could not lazily evaluate deferred parameter: %v", err) + } + p.values[name] = value + return value, nil + } + return "", nil +} diff --git a/pkg/steps/template_test.go b/pkg/api/parameters_test.go similarity index 90% rename from pkg/steps/template_test.go rename to pkg/api/parameters_test.go index 1880d8e6..cd78e055 100644 --- a/pkg/steps/template_test.go +++ b/pkg/api/parameters_test.go @@ -1,14 +1,22 @@ -package steps +package api import ( "reflect" "testing" "k8s.io/apimachinery/pkg/util/diff" - - "github.com/openshift/ci-operator/pkg/api" ) +func someStepLink(as string) StepLink { + return ExternalImageLink(ImageStreamTagReference{ + Cluster: "cluster.com", + Namespace: "namespace", + Name: "name", + Tag: "tag", + As: as, + }) +} + func TestDeferredParametersAllLinks(t *testing.T) { var testCases = []struct { purpose string @@ -17,7 +25,7 @@ func TestDeferredParametersAllLinks(t *testing.T) { }{{ purpose: "AllLinks should return a slice with all links for all names", dp: &DeferredParameters{ - links: map[string][]api.StepLink{ + links: map[string][]StepLink{ "K1": {someStepLink("ONE"), someStepLink("TWO")}, "K2": {someStepLink("THREE")}, }, @@ -93,11 +101,11 @@ func TestDeferredParametersAddHasLinksGet(t *testing.T) { dp *DeferredParameters callAdd bool name string - link api.StepLink + link StepLink fn func() (string, error) expectedHas bool - expectedLinks []api.StepLink + expectedLinks []StepLink expectedGet string }{{ purpose: "After `Add(key, link, f)`: Has(key)->true, Links(key)->{link}, Get(key)->f()", @@ -108,7 +116,7 @@ func TestDeferredParametersAddHasLinksGet(t *testing.T) { fn: func() (string, error) { return "value", nil }, expectedHas: true, - expectedLinks: []api.StepLink{someStepLink("name")}, + expectedLinks: []StepLink{someStepLink("name")}, expectedGet: "value", }, { purpose: "Without Add(): Has(key)->false and Links(key)->nil", @@ -124,9 +132,9 @@ func TestDeferredParametersAddHasLinksGet(t *testing.T) { }, { purpose: "After `Add(key, new-link)` when `key` already present: Has(key)->true and Links(key)->{new-link}", dp: &DeferredParameters{ - fns: api.ParameterMap{"key": func() (string, error) { return "old", nil }}, + fns: ParameterMap{"key": func() (string, error) { return "old", nil }}, values: map[string]string{}, - links: map[string][]api.StepLink{"key": {someStepLink("old-link")}}, + links: map[string][]StepLink{"key": {someStepLink("old-link")}}, }, callAdd: true, name: "key", @@ -134,7 +142,7 @@ func TestDeferredParametersAddHasLinksGet(t *testing.T) { fn: func() (string, error) { return "new", nil }, expectedHas: true, - expectedLinks: []api.StepLink{someStepLink("new-link")}, + expectedLinks: []StepLink{someStepLink("new-link")}, expectedGet: "new", }} for _, tc := range testCases { @@ -175,9 +183,9 @@ func TestDeferredParametersGetSet(t *testing.T) { }, { purpose: "Existing key is not overwritten", dp: &DeferredParameters{ - fns: make(api.ParameterMap), + fns: make(ParameterMap), values: map[string]string{"key": "oldValue"}, - links: map[string][]api.StepLink{}, + links: map[string][]StepLink{}, }, name: "key", callSet: true, @@ -192,7 +200,7 @@ func TestDeferredParametersGetSet(t *testing.T) { "key": func() (string, error) { return "lazyValue", nil }, }, values: map[string]string{}, - links: map[string][]api.StepLink{}, + links: map[string][]StepLink{}, }, name: "key", callSet: true, diff --git a/pkg/api/types.go b/pkg/api/types.go index 01a96e94..134d974e 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -529,4 +529,6 @@ const ( RPMServeLocation = "/srv/repo" StableImageStream = "stable" + + ComponentFormatReplacement = "${component}" ) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index dc0c0974..d00ab205 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -94,7 +94,7 @@ func FromConfig( podClient = steps.NewPodClient(coreGetter, clusterConfig, coreGetter.RESTClient()) } - params := steps.NewDeferredParameters() + params := api.NewDeferredParameters() params.Add("JOB_NAME", nil, func() (string, error) { return jobSpec.Job, nil }) params.Add("JOB_NAME_HASH", nil, func() (string, error) { return fmt.Sprintf("%x", sha256.Sum256([]byte(jobSpec.Job)))[:5], nil }) params.Add("JOB_NAME_SAFE", nil, func() (string, error) { return strings.Replace(jobSpec.Job, "_", "-", -1), nil }) @@ -138,7 +138,7 @@ func FromConfig( if err != nil { return nil, nil, fmt.Errorf("unable to access release images on remote cluster: %v", err) } - step = steps.ReleaseImagesTagStep(*rawStep.ReleaseImagesTagStepConfiguration, srcClient, imageClient, routeGetter, configMapGetter, params, jobSpec) + step = release.ReleaseImagesTagStep(*rawStep.ReleaseImagesTagStepConfiguration, srcClient, imageClient, routeGetter, configMapGetter, params, jobSpec) stepLinks = append(stepLinks, step.Creates()...) releaseStep = release.AssembleReleaseStep(*rawStep.ReleaseImagesTagStepConfiguration, config.Resources, podClient, imageClient, artifactDir, jobSpec) @@ -167,7 +167,7 @@ func FromConfig( releaseStep, _ = checkForFullyQualifiedStep(releaseStep, params) buildSteps = append(buildSteps, releaseStep) } else { - buildSteps = append(buildSteps, steps.StableImagesTagStep(imageClient, jobSpec)) + buildSteps = append(buildSteps, release.StableImagesTagStep(imageClient, jobSpec)) } buildSteps = append(buildSteps, steps.ImagesReadyStep(imageStepLinks)) @@ -184,7 +184,7 @@ func FromConfig( tags = append(tags, string(image.To)) } } - postSteps = append(postSteps, steps.PromotionStep(*cfg, tags, imageClient, imageClient, jobSpec)) + postSteps = append(postSteps, release.PromotionStep(*cfg, tags, imageClient, imageClient, jobSpec)) } return buildSteps, postSteps, nil @@ -193,7 +193,7 @@ func FromConfig( // checkForFullyQualifiedStep if all output parameters of this step are part of the // environment, replace the step with a shim that automatically provides those variables. // Returns true if the step was replaced. -func checkForFullyQualifiedStep(step api.Step, params *steps.DeferredParameters) (api.Step, bool) { +func checkForFullyQualifiedStep(step api.Step, params *api.DeferredParameters) (api.Step, bool) { provides, link := step.Provides() if values, ok := envHasAllParameters(provides); ok { diff --git a/pkg/steps/release/create_release.go b/pkg/steps/release/create_release.go index a72fe69c..d2e00e6a 100644 --- a/pkg/steps/release/create_release.go +++ b/pkg/steps/release/create_release.go @@ -37,12 +37,12 @@ func (s *assembleReleaseStep) Run(ctx context.Context, dry bool) error { if err != nil { return fmt.Errorf("could not resolve stable imagestream: %v", err) } - cvo, ok := resolvePullSpec(stable, "cluster-version-operator") + cvo, ok := resolvePullSpec(stable, "cluster-version-operator", true) if !ok { log.Printf("No release image necessary, stable image stream does not include a cluster-version-operator image") return nil } - if _, ok := resolvePullSpec(stable, "cli"); !ok { + if _, ok := resolvePullSpec(stable, "cli", true); !ok { return fmt.Errorf("no 'cli' image was tagged into the stable stream, that image is required for building a release") } @@ -148,24 +148,3 @@ func AssembleReleaseStep(config api.ReleaseTagConfiguration, resources api.Resou jobSpec: jobSpec, } } - -func resolvePullSpec(is *imageapi.ImageStream, tag string) (string, bool) { - for _, tags := range is.Status.Tags { - if tags.Tag != tag { - continue - } - if len(tags.Items) == 0 { - break - } - if image := tags.Items[0].Image; len(image) > 0 { - if len(is.Status.PublicDockerImageRepository) > 0 { - return fmt.Sprintf("%s@%s", is.Status.PublicDockerImageRepository, image), true - } - if len(is.Status.DockerImageRepository) > 0 { - return fmt.Sprintf("%s@%s", is.Status.DockerImageRepository, image), true - } - } - break - } - return "", false -} diff --git a/pkg/steps/promote.go b/pkg/steps/release/promote.go similarity index 99% rename from pkg/steps/promote.go rename to pkg/steps/release/promote.go index ab62a727..59610994 100644 --- a/pkg/steps/promote.go +++ b/pkg/steps/release/promote.go @@ -1,4 +1,4 @@ -package steps +package release import ( "context" diff --git a/pkg/steps/release_images.go b/pkg/steps/release/release_images.go similarity index 83% rename from pkg/steps/release_images.go rename to pkg/steps/release/release_images.go index 113352a3..62429578 100644 --- a/pkg/steps/release_images.go +++ b/pkg/steps/release/release_images.go @@ -1,4 +1,4 @@ -package steps +package release import ( "context" @@ -19,8 +19,6 @@ import ( const ( ConfigMapName = "release" - - componentFormatReplacement = "${component}" ) // stableImagesTagStep is used when no release configuration is necessary @@ -92,7 +90,7 @@ type releaseImagesTagStep struct { dstClient imageclientset.ImageV1Interface routeClient routeclientset.RoutesGetter configMapClient coreclientset.ConfigMapsGetter - params *DeferredParameters + params *api.DeferredParameters jobSpec *api.JobSpec } @@ -206,7 +204,7 @@ func (s *releaseImagesTagStep) Run(ctx context.Context, dry bool) error { } for _, tag := range is.Spec.Tags { - spec, ok := resolvePullSpec(is, tag.Name) + spec, ok := resolvePullSpec(is, tag.Name, false) if !ok { continue } @@ -290,7 +288,7 @@ func (s *releaseImagesTagStep) Run(ctx context.Context, dry bool) error { return fmt.Errorf("could not copy stable imagestreamtag: %v", err) } - if spec, ok := resolvePullSpec(&stableImageStream, tag.Name); ok { + if spec, ok := resolvePullSpec(&stableImageStream, tag.Name, false); ok { s.params.Set(componentToParamName(tag.Name), spec) } } @@ -300,66 +298,8 @@ func (s *releaseImagesTagStep) Run(ctx context.Context, dry bool) error { return nil } -func (s *releaseImagesTagStep) createReleaseConfigMap(dry bool) error { - imageBase := "dry-fake" - rpmRepo := "dry-fake" - if !dry { - originImageStream, err := s.dstClient.ImageStreams(s.jobSpec.Namespace).Get("origin", meta.GetOptions{}) - if err != nil { - return fmt.Errorf("could not resolve main release ImageStream: %v", err) - } - if len(originImageStream.Status.PublicDockerImageRepository) == 0 { - return fmt.Errorf("release ImageStream %s/%s is not exposed externally", originImageStream.Namespace, originImageStream.Name) - } - imageBase = originImageStream.Status.PublicDockerImageRepository - - rpmRepoServer, err := s.routeClient.Routes(s.jobSpec.Namespace).Get(RPMRepoName, meta.GetOptions{}) - if !errors.IsNotFound(err) { - return fmt.Errorf("could not retrieve RPM repo server route: %v", err) - } else { - rpmRepoServer, err = s.routeClient.Routes(s.config.Namespace).Get(RPMRepoName, meta.GetOptions{}) - if err != nil { - return fmt.Errorf("could not retrieve RPM repo server route: %v", err) - } - } - rpmRepo = rpmRepoServer.Spec.Host - } - - cm := &coreapi.ConfigMap{ - ObjectMeta: meta.ObjectMeta{ - Name: ConfigMapName, - Namespace: s.jobSpec.Namespace, - }, - Data: map[string]string{ - "image-base": imageBase, - "rpm-repo": rpmRepo, - }, - } - if dry { - cmJSON, err := json.MarshalIndent(cm, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal configmap: %v", err) - } - fmt.Printf("%s\n", cmJSON) - return nil - } - if _, err := s.configMapClient.ConfigMaps(s.jobSpec.Namespace).Create(cm); err != nil && !errors.IsAlreadyExists(err) { - return fmt.Errorf("could not create release configmap: %v", err) - } - return nil -} - func (s *releaseImagesTagStep) Done() (bool, error) { - log.Printf("Checking for existence of %s ConfigMap", ConfigMapName) - if _, err := s.configMapClient.ConfigMaps(s.jobSpec.Namespace).Get(ConfigMapName, meta.GetOptions{}); err != nil { - if errors.IsNotFound(err) { - return false, nil - } else { - return false, fmt.Errorf("could not retrieve release configmap: %v", err) - } - } else { - return true, nil - } + return false, nil } func (s *releaseImagesTagStep) Requires() []api.StepLink { @@ -384,9 +324,9 @@ func (s *releaseImagesTagStep) imageFormat() (string, error) { registry := strings.SplitN(spec, "/", 2)[0] var format string if len(s.config.Name) > 0 { - format = fmt.Sprintf("%s/%s/%s:%s", registry, s.jobSpec.Namespace, fmt.Sprintf("%s%s", s.config.NamePrefix, api.StableImageStream), componentFormatReplacement) + format = fmt.Sprintf("%s/%s/%s:%s", registry, s.jobSpec.Namespace, fmt.Sprintf("%s%s", s.config.NamePrefix, api.StableImageStream), api.ComponentFormatReplacement) } else { - format = fmt.Sprintf("%s/%s/%s:%s", registry, s.jobSpec.Namespace, fmt.Sprintf("%s%s", s.config.NamePrefix, componentFormatReplacement), s.config.Tag) + format = fmt.Sprintf("%s/%s/%s:%s", registry, s.jobSpec.Namespace, fmt.Sprintf("%s%s", s.config.NamePrefix, api.ComponentFormatReplacement), s.config.Tag) } return format, nil } @@ -411,7 +351,7 @@ func (s *releaseImagesTagStep) Description() string { return fmt.Sprintf("Find all of the input images from %s and tag them into the output image stream", sourceName(s.config)) } -func ReleaseImagesTagStep(config api.ReleaseTagConfiguration, srcClient, dstClient imageclientset.ImageV1Interface, routeClient routeclientset.RoutesGetter, configMapClient coreclientset.ConfigMapsGetter, params *DeferredParameters, jobSpec *api.JobSpec) api.Step { +func ReleaseImagesTagStep(config api.ReleaseTagConfiguration, srcClient, dstClient imageclientset.ImageV1Interface, routeClient routeclientset.RoutesGetter, configMapClient coreclientset.ConfigMapsGetter, params *api.DeferredParameters, jobSpec *api.JobSpec) api.Step { // when source and destination client are the same, we don't need to use external imports if srcClient == dstClient { config.Cluster = "" @@ -431,7 +371,7 @@ func componentToParamName(component string) string { return strings.ToUpper(strings.Replace(component, "-", "_", -1)) } -func resolvePullSpec(is *imageapi.ImageStream, tag string) (string, bool) { +func resolvePullSpec(is *imageapi.ImageStream, tag string, requireExact bool) (string, bool) { for _, tags := range is.Status.Tags { if tags.Tag != tag { continue @@ -449,6 +389,9 @@ func resolvePullSpec(is *imageapi.ImageStream, tag string) (string, bool) { } break } + if requireExact { + return "", false + } if len(is.Status.PublicDockerImageRepository) > 0 { return fmt.Sprintf("%s:%s", is.Status.PublicDockerImageRepository, tag), true } diff --git a/pkg/steps/template.go b/pkg/steps/template.go index d3569f72..a859d2df 100644 --- a/pkg/steps/template.go +++ b/pkg/steps/template.go @@ -10,7 +10,6 @@ import ( "path/filepath" "sort" "strings" - "sync" "time" "k8s.io/client-go/rest" @@ -31,7 +30,7 @@ import ( type templateExecutionStep struct { template *templateapi.Template - params *DeferredParameters + params *api.DeferredParameters templateClient TemplateClient podClient PodClient artifactDir string @@ -77,7 +76,7 @@ func (s *templateExecutionStep) Run(ctx context.Context, dry bool) error { if err != nil { return fmt.Errorf("could not resolve image format: %v", err) } - s.template.Parameters[i].Value = strings.Replace(format, componentFormatReplacement, component, -1) + s.template.Parameters[i].Value = strings.Replace(format, api.ComponentFormatReplacement, component, -1) } } } @@ -230,7 +229,7 @@ func (s *templateExecutionStep) Description() string { return fmt.Sprintf("Run template %s", s.template.Name) } -func TemplateExecutionStep(template *templateapi.Template, params *DeferredParameters, podClient PodClient, templateClient TemplateClient, artifactDir string, jobSpec *api.JobSpec) api.Step { +func TemplateExecutionStep(template *templateapi.Template, params *api.DeferredParameters, podClient PodClient, templateClient TemplateClient, artifactDir string, jobSpec *api.JobSpec) api.Step { return &templateExecutionStep{ template: template, params: params, @@ -241,115 +240,6 @@ func TemplateExecutionStep(template *templateapi.Template, params *DeferredParam } } -type DeferredParameters struct { - lock sync.Mutex - fns api.ParameterMap - values map[string]string - links map[string][]api.StepLink -} - -func NewDeferredParameters() *DeferredParameters { - return &DeferredParameters{ - fns: make(api.ParameterMap), - values: make(map[string]string), - links: make(map[string][]api.StepLink), - } -} - -func (p *DeferredParameters) Map() (map[string]string, error) { - p.lock.Lock() - defer p.lock.Unlock() - m := make(map[string]string) - for k, fn := range p.fns { - if v, ok := p.values[k]; ok { - m[k] = v - continue - } - v, err := fn() - if err != nil { - return nil, fmt.Errorf("could not lazily evaluate deferred parameter: %v", err) - } - p.values[k] = v - m[k] = v - } - return m, nil -} - -func (p *DeferredParameters) Set(name, value string) { - p.lock.Lock() - defer p.lock.Unlock() - if _, ok := p.fns[name]; ok { - return - } - if _, ok := p.values[name]; ok { - return - } - p.values[name] = value -} - -func (p *DeferredParameters) Add(name string, link api.StepLink, fn func() (string, error)) { - p.lock.Lock() - defer p.lock.Unlock() - p.fns[name] = fn - if link != nil { - p.links[name] = []api.StepLink{link} - } -} - -func (p *DeferredParameters) Has(name string) bool { - p.lock.Lock() - defer p.lock.Unlock() - _, ok := p.fns[name] - if ok { - return true - } - _, ok = os.LookupEnv(name) - return ok -} - -func (p *DeferredParameters) Links(name string) []api.StepLink { - p.lock.Lock() - defer p.lock.Unlock() - if _, ok := os.LookupEnv(name); ok { - return nil - } - return p.links[name] -} - -func (p *DeferredParameters) AllLinks() []api.StepLink { - p.lock.Lock() - defer p.lock.Unlock() - var links []api.StepLink - for name, v := range p.links { - if _, ok := os.LookupEnv(name); ok { - continue - } - links = append(links, v...) - } - return links -} - -func (p *DeferredParameters) Get(name string) (string, error) { - p.lock.Lock() - defer p.lock.Unlock() - if value, ok := p.values[name]; ok { - return value, nil - } - if value, ok := os.LookupEnv(name); ok { - p.values[name] = value - return value, nil - } - if fn, ok := p.fns[name]; ok { - value, err := fn() - if err != nil { - return "", fmt.Errorf("could not lazily evaluate deferred parameter: %v", err) - } - p.values[name] = value - return value, nil - } - return "", nil -} - type TemplateClient interface { templateclientset.TemplateV1Interface Process(namespace string, template *templateapi.Template) (*templateapi.Template, error) diff --git a/pkg/steps/write_params.go b/pkg/steps/write_params.go index 0f3abb15..72d12614 100644 --- a/pkg/steps/write_params.go +++ b/pkg/steps/write_params.go @@ -13,7 +13,7 @@ import ( ) type writeParametersStep struct { - params *DeferredParameters + params *api.DeferredParameters paramFile string } @@ -69,7 +69,7 @@ func (s *writeParametersStep) Name() string { return "parameters/write" } func (s *writeParametersStep) Description() string { return "Write the job parameters to disk" } -func WriteParametersStep(params *DeferredParameters, paramFile string) api.Step { +func WriteParametersStep(params *api.DeferredParameters, paramFile string) api.Step { return &writeParametersStep{ params: params, paramFile: paramFile, diff --git a/pkg/steps/write_params_test.go b/pkg/steps/write_params_test.go index d040dd9b..c80e358c 100644 --- a/pkg/steps/write_params_test.go +++ b/pkg/steps/write_params_test.go @@ -10,7 +10,7 @@ import ( ) func TestWriteParamsStep(t *testing.T) { - params := NewDeferredParameters() + params := api.NewDeferredParameters() params.Add("K1", someStepLink("another-step"), func() (string, error) { return "V1", nil }) params.Add("K2", someStepLink("another-step"), func() (string, error) { return "V:2", nil }) paramFile, err := ioutil.TempFile("", "")