From 1948327ba1e3cbc6b14ac5ef19a20149b758cc15 Mon Sep 17 00:00:00 2001 From: Sally O'Malley Date: Mon, 20 Apr 2020 10:44:59 -0400 Subject: [PATCH 1/2] Bug 1823143: oc adm release - use ImageContentSourcePolicy if found check image reference, icsp, then image instead of only using image references for 'oc adm release ...' commands 0. If user passes --icsp-file path, fail if no valid sources found from the file 1. Try the current flow of lookup image from any underlying image references. If this fails, go to 2. 2. Try to gather image source info from ImageContentSourcePolicy, if this fails go to 3. 3. Set the registry/repo/name to be that of user-given release rather than its refs. If image not found, return the original error from 1. When working with mirrored release payloads, a release from a mirrored registry, mylocalregistry/ocp/release:4.5.0-0.nightly-2020-04-18-093630 mirrored from registry.svc.ci.openshift.org/ocp/release:4.5.0-0.nightly-2020-04-18-093630 - Both reference 'quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2eb0a51...'. In case of disconnected, oc will use 'mylocalregistry/ocp/release' instead of 'quay.io/openshift-release-dev/ocp-v4.0-art-dev' _or_ will get image source information from ICSP in cluster. Also, `oc adm release mirror` will write ICSP file to local disk. --- contrib/completions/bash/oc | 15 ++ contrib/completions/zsh/oc | 15 ++ pkg/cli/admin/catalog/build.go | 2 +- pkg/cli/admin/catalog/mirror.go | 6 +- pkg/cli/admin/release/extract.go | 29 ++- pkg/cli/admin/release/extract_tools.go | 255 +++++++++++-------- pkg/cli/admin/release/info.go | 135 +++++++++- pkg/cli/admin/release/mirror.go | 145 +++++++---- pkg/cli/image/append/append.go | 20 +- pkg/cli/image/extract/extract.go | 27 +- pkg/cli/image/image.go | 2 +- pkg/cli/image/imagesource/options.go | 62 ++++- pkg/cli/image/info/info.go | 14 +- pkg/cli/image/manifest/.security_test.go.swp | Bin 0 -> 16384 bytes pkg/cli/image/manifest/manifest.go | 74 +----- pkg/cli/image/manifest/security.go | 222 ++++++++++++++++ pkg/cli/image/manifest/security_test.go | 204 +++++++++++++++ pkg/cli/image/mirror/mirror.go | 28 +- 18 files changed, 955 insertions(+), 300 deletions(-) create mode 100644 pkg/cli/image/manifest/.security_test.go.swp create mode 100644 pkg/cli/image/manifest/security.go create mode 100644 pkg/cli/image/manifest/security_test.go diff --git a/contrib/completions/bash/oc b/contrib/completions/bash/oc index a25ada03b3..4243d368ac 100644 --- a/contrib/completions/bash/oc +++ b/contrib/completions/bash/oc @@ -5444,6 +5444,9 @@ _oc_adm_release_extract() flags+=("--git=") two_word_flags+=("--git") local_nonpersistent_flags+=("--git=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--max-per-registry=") @@ -5545,6 +5548,9 @@ _oc_adm_release_info() flags+=("--dir=") two_word_flags+=("--dir") local_nonpersistent_flags+=("--dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image-for=") two_word_flags+=("--image-for") local_nonpersistent_flags+=("--image-for=") @@ -5646,6 +5652,9 @@ _oc_adm_release_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--max-per-registry=") @@ -5657,6 +5666,9 @@ _oc_adm_release_mirror() two_word_flags+=("--registry-config") two_word_flags+=("-a") local_nonpersistent_flags+=("--registry-config=") + flags+=("--release-image-icsp-to-dir=") + two_word_flags+=("--release-image-icsp-to-dir") + local_nonpersistent_flags+=("--release-image-icsp-to-dir=") flags+=("--release-image-signature-to-dir=") two_word_flags+=("--release-image-signature-to-dir") local_nonpersistent_flags+=("--release-image-signature-to-dir=") @@ -13501,6 +13513,9 @@ _oc_image_extract() flags+=("--filter-by-os=") two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--only-files") diff --git a/contrib/completions/zsh/oc b/contrib/completions/zsh/oc index fc99cb14cd..5ebd35740f 100644 --- a/contrib/completions/zsh/oc +++ b/contrib/completions/zsh/oc @@ -5544,6 +5544,9 @@ _oc_adm_release_extract() flags+=("--git=") two_word_flags+=("--git") local_nonpersistent_flags+=("--git=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--max-per-registry=") @@ -5645,6 +5648,9 @@ _oc_adm_release_info() flags+=("--dir=") two_word_flags+=("--dir") local_nonpersistent_flags+=("--dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image-for=") two_word_flags+=("--image-for") local_nonpersistent_flags+=("--image-for=") @@ -5746,6 +5752,9 @@ _oc_adm_release_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--max-per-registry=") @@ -5757,6 +5766,9 @@ _oc_adm_release_mirror() two_word_flags+=("--registry-config") two_word_flags+=("-a") local_nonpersistent_flags+=("--registry-config=") + flags+=("--release-image-icsp-to-dir=") + two_word_flags+=("--release-image-icsp-to-dir") + local_nonpersistent_flags+=("--release-image-icsp-to-dir=") flags+=("--release-image-signature-to-dir=") two_word_flags+=("--release-image-signature-to-dir") local_nonpersistent_flags+=("--release-image-signature-to-dir=") @@ -13601,6 +13613,9 @@ _oc_image_extract() flags+=("--filter-by-os=") two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--only-files") diff --git a/pkg/cli/admin/catalog/build.go b/pkg/cli/admin/catalog/build.go index d66a277c65..fbe95689e9 100644 --- a/pkg/cli/admin/catalog/build.go +++ b/pkg/cli/admin/catalog/build.go @@ -96,7 +96,7 @@ func (o *BuildImageOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, arg } imageFromRealeaseTags := func(img string) { - info, err := infoOpts.LoadReleaseInfo(img, false) + info, err := infoOpts.LoadReleaseInfo(img, false, false, "") if err != nil { klog.V(2).Infof("unable to load image from %s: %v", img, err) return diff --git a/pkg/cli/admin/catalog/mirror.go b/pkg/cli/admin/catalog/mirror.go index 54ea5764ca..00bfe74fce 100644 --- a/pkg/cli/admin/catalog/mirror.go +++ b/pkg/cli/admin/catalog/mirror.go @@ -115,7 +115,7 @@ func NewMirrorCatalog(f kcmdutil.Factory, streams genericclioptions.IOStreams) * Long: mirrorLong, Example: mirrorExample, Run: func(cmd *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(cmd, args)) + kcmdutil.CheckErr(o.Complete(f, cmd, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -137,7 +137,7 @@ func NewMirrorCatalog(f kcmdutil.Factory, streams genericclioptions.IOStreams) * return cmd } -func (o *MirrorCatalogOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *MirrorCatalogOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if len(args) < 2 { return fmt.Errorf("must specify source and dest") } @@ -291,7 +291,7 @@ func (o *MirrorCatalogOptions) Complete(cmd *cobra.Command, args []string) error } e.Paths = []string{o.DatabasePath} e.Confirm = true - if err := e.Complete(cmd, []string{o.SourceRef.String()}); err != nil { + if err := e.Complete(f, cmd, []string{o.SourceRef.String()}); err != nil { return "", err } if err := e.Validate(); err != nil { diff --git a/pkg/cli/admin/release/extract.go b/pkg/cli/admin/release/extract.go index bca93fa435..6a791e16be 100644 --- a/pkg/cli/admin/release/extract.go +++ b/pkg/cli/admin/release/extract.go @@ -102,6 +102,7 @@ func NewExtract(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra. } flags := cmd.Flags() o.SecurityOptions.Bind(flags) + o.FilterOptions.Bind(flags) o.ParallelOptions.Bind(flags) flags.StringVar(&o.From, "from", o.From, "Image containing the release payload.") @@ -127,7 +128,9 @@ type ExtractOptions struct { genericclioptions.IOStreams SecurityOptions imagemanifest.SecurityOptions + FilterOptions imagemanifest.FilterOptions ParallelOptions imagemanifest.ParallelOptions + InfoOptions InfoOptions Output string @@ -174,6 +177,15 @@ func (o *ExtractOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args [ } o.From = args[0] + if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { + return err + } + if err := o.InfoOptions.Complete(f, cmd, args); err != nil { + return err + } + if err := o.SecurityOptions.Complete(f, o.From); err != nil { + return err + } return nil } @@ -208,6 +220,13 @@ func (o *ExtractOptions) Run() error { } } + opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) + opts.ParallelOptions = o.ParallelOptions + opts.FilterOptions = o.FilterOptions + opts.SecurityOptions = o.SecurityOptions + + opts.FileDir = o.FileDir + switch { case sources > 1: return fmt.Errorf("only one of --tools, --command, --credentials-requests, --file, or --git may be specified") @@ -234,11 +253,6 @@ func (o *ExtractOptions) Run() error { if err != nil { return err } - opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) - opts.ParallelOptions = o.ParallelOptions - opts.SecurityOptions = o.SecurityOptions - opts.FileDir = o.FileDir - switch { case len(o.File) > 0: if o.ImageMetadataCallback != nil { @@ -419,10 +433,7 @@ func (o *ExtractOptions) extractGit(dir string) error { return err } - opts := NewInfoOptions(o.IOStreams) - opts.SecurityOptions = o.SecurityOptions - opts.FileDir = o.FileDir - release, err := opts.LoadReleaseInfo(o.From, false) + release, err := o.InfoOptions.LoadReleaseInfo(o.From, false, false, "") if err != nil { return err } diff --git a/pkg/cli/admin/release/extract_tools.go b/pkg/cli/admin/release/extract_tools.go index 5e1121a6f5..766a9d8eaa 100644 --- a/pkg/cli/admin/release/extract_tools.go +++ b/pkg/cli/admin/release/extract_tools.go @@ -292,21 +292,154 @@ func (o *ExtractOptions) extractCommand(command string) error { } } - // load the release image + // load release image dir := o.Directory - infoOptions := NewInfoOptions(o.IOStreams) - infoOptions.SecurityOptions = o.SecurityOptions - infoOptions.FileDir = o.FileDir - release, err := infoOptions.LoadReleaseInfo(o.From, false) - if err != nil { - return err + var releaseFromRefErr error + targetRelease := targetReleaseInfo{} + if len(o.SecurityOptions.ImageContentSourcePolicyFile) > 0 { + o.SecurityOptions.AddImageSourcePoliciesFromFile(o.From) + releaseFromImageSources, err := o.InfoOptions.LoadReleaseInfo(o.From, false, true, o.SecurityOptions.ImageContentSourcePolicyFile) + if err != nil { + return fmt.Errorf("could not load release %s from icsp file %s: %v", o.From, o.SecurityOptions.ImageContentSourcePolicyFile, err) + } + targetRelease, err = o.setReleaseLookup(releaseFromImageSources, targets, currentOS, willArchive) + if err != nil { + // if icsp-file set, then return error if lookup from icsp fails + return fmt.Errorf("failed lookup of release %s from icsp file %s: %v", o.From, o.SecurityOptions.ImageContentSourcePolicyFile, err) + } + } else { + // This will be tried first, if no ImageContentSourcePolicyFile passed + releaseFromRef, loadReleaseFromRefErr := o.InfoOptions.LoadReleaseInfo(o.From, false, false, "") + if loadReleaseFromRefErr == nil { + targetRelease, releaseFromRefErr = o.setReleaseLookup(releaseFromRef, targets, currentOS, willArchive) + // This will be returned if further lookup fails + if releaseFromRefErr != nil { + // If there's an error, now look for other imageSources - icsp and/or the user-passed image registry/repo/name + klog.V(2).Infof("Failed lookup of release from its reference: %v", releaseFromRefErr) + // now try other sources + releaseFromImageSources, err := o.InfoOptions.LoadReleaseInfo(o.From, false, true, "") + if err != nil { + return err + } + targetRelease, err = o.setReleaseLookup(releaseFromImageSources, targets, currentOS, willArchive) + if err != nil { + klog.V(2).Infof("Failed lookup of release: %v", err) + return releaseFromRefErr + } + } + } else { + klog.V(2).Infof("Failed to load release info from image reference: %v", loadReleaseFromRefErr) + // now try other sources + releaseFromImageSources, err := o.InfoOptions.LoadReleaseInfo(o.From, false, true, "") + if err != nil { + return err + } + targetRelease, err = o.setReleaseLookup(releaseFromImageSources, targets, currentOS, willArchive) + if err != nil { + klog.V(2).Infof("Failed lookup of release: %v", err) + return loadReleaseFromRefErr + } + } + } + if targetRelease.willArchive { + buf := &bytes.Buffer{} + fmt.Fprintf(buf, heredoc.Doc(` + Client tools for OpenShift + -------------------------- + + These archives contain the client tooling for [OpenShift](https://docs.openshift.com). + + To verify the contents of this directory, use the 'gpg' and 'shasum' tools to + ensure the archives you have downloaded match those published from this location. + + The openshift-install binary has been preconfigured to install the following release: + + --- + + `)) + if err := describeReleaseInfo(buf, targetRelease.release, false, false, true, false); err != nil { + return err + } + filename := "release.txt" + if err := ioutil.WriteFile(filepath.Join(dir, filename), buf.Bytes(), 0644); err != nil { + return err + } + hash := hashFn() + hash.Write(buf.Bytes()) + targetRelease.hashByTargetName[filename] = hex.EncodeToString(hash.Sum(nil)) } + + // write a checksum of the tar files to disk as sha256sum.txt.asc + if len(targetRelease.hashByTargetName) > 0 { + var keys []string + for k := range targetRelease.hashByTargetName { + keys = append(keys, k) + } + sort.Strings(keys) + var lines []string + for _, k := range keys { + hash := targetRelease.hashByTargetName[k] + lines = append(lines, fmt.Sprintf("%s %s", hash, filepath.Base(k))) + } + // ensure a trailing newline + if len(lines[len(lines)-1]) != 0 { + lines = append(lines, "") + } + // write the content manifest + data := []byte(strings.Join(lines, "\n")) + filename := "sha256sum.txt" + if err := ioutil.WriteFile(filepath.Join(dir, filename), data, 0644); err != nil { + return fmt.Errorf("unable to write checksum file: %v", err) + } + // sign the content manifest + if signer != nil { + buf := &bytes.Buffer{} + if err := openpgp.ArmoredDetachSign(buf, signer, bytes.NewBuffer(data), nil); err != nil { + return fmt.Errorf("unable to sign the sha256sum.txt file: %v", err) + } + if err := ioutil.WriteFile(filepath.Join(dir, filename+".asc"), buf.Bytes(), 0644); err != nil { + return fmt.Errorf("unable to write signed manifest: %v", err) + } + } + } + + // if we did not process some targets, report that to the user and error if necessary + if len(targetRelease.targetsByName) > 0 { + var missing []string + for _, target := range targetRelease.targetsByName { + missing = append(missing, target.Mapping.From) + } + sort.Strings(missing) + if len(missing) == 1 { + return fmt.Errorf("image did not contain %s", missing[0]) + } + return fmt.Errorf("unable to find multiple files: %s", strings.Join(missing, ", ")) + } + + return nil +} + +type targetReleaseInfo struct { + release *ReleaseInfo + willArchive bool + hashByTargetName map[string]string + targetsByName map[string]extractTarget +} + +func (o *ExtractOptions) setReleaseLookup(release *ReleaseInfo, targets []extractTarget, currentOS string, toArchive bool) (targetReleaseInfo, error) { releaseName := release.PreferredName() refExact := release.ImageRef refExact.Ref.Tag = "" refExact.Ref.ID = release.Digest.String() exactReleaseImage := refExact.String() + tr := targetReleaseInfo{ + release: release, + willArchive: toArchive, + hashByTargetName: make(map[string]string), + targetsByName: make(map[string]extractTarget), + } + // resolve target image references to their pull specs missing := sets.NewString() var validTargets []extractTarget @@ -323,16 +456,16 @@ func (o *ExtractOptions) extractCommand(command string) error { klog.V(2).Infof("Will extract %s from %s", target.Mapping.From, spec) ref, err := imagereference.Parse(spec) if err != nil { - return err + return targetReleaseInfo{}, err } target.Mapping.Image = spec target.Mapping.ImageRef = imagesource.TypedImageReference{Ref: ref, Type: imagesource.DestinationRegistry} if target.AsArchive { - willArchive = true + tr.willArchive = true target.Mapping.Name = fmt.Sprintf(target.ArchiveFormat, releaseName) - target.Mapping.To = filepath.Join(dir, target.Mapping.Name) + target.Mapping.To = filepath.Join(o.Directory, target.Mapping.Name) } else { - target.Mapping.To = filepath.Join(dir, target.Command) + target.Mapping.To = filepath.Join(o.Directory, target.Command) target.Mapping.Name = fmt.Sprintf("%s-%s", target.OS, target.Command) } validTargets = append(validTargets, target) @@ -340,9 +473,9 @@ func (o *ExtractOptions) extractCommand(command string) error { if len(validTargets) == 0 { if len(missing) == 1 { - return fmt.Errorf("the image %q containing the desired command is not available", missing.List()[0]) + return targetReleaseInfo{}, fmt.Errorf("the image %q containing the desired command is not available", missing.List()[0]) } - return fmt.Errorf("some required images are missing: %s", strings.Join(missing.List(), ", ")) + return targetReleaseInfo{}, fmt.Errorf("some required images are missing: %s", strings.Join(missing.List(), ", ")) } if len(missing) > 0 { fmt.Fprintf(o.ErrOut, "warning: Some commands can not be extracted due to missing images: %s\n", strings.Join(missing.List(), ", ")) @@ -356,16 +489,13 @@ func (o *ExtractOptions) extractCommand(command string) error { // create the mapping lookup of the valid targets var extractLock sync.Mutex - targetsByName := make(map[string]extractTarget) for _, target := range validTargets { - targetsByName[target.Mapping.Name] = target + tr.targetsByName[target.Mapping.Name] = target opts.Mappings = append(opts.Mappings, target.Mapping) } - hashByTargetName := make(map[string]string) - // ensure to is a directory - if err := os.MkdirAll(dir, 0777); err != nil { - return err + if err := os.MkdirAll(o.Directory, 0777); err != nil { + return targetReleaseInfo{}, err } // as each layer is extracted, take the output binary and write it to disk @@ -374,7 +504,7 @@ func (o *ExtractOptions) extractCommand(command string) error { target, ok := func() (extractTarget, bool) { extractLock.Lock() defer extractLock.Unlock() - target, ok := targetsByName[layer.Mapping.Name] + target, ok := tr.targetsByName[layer.Mapping.Name] return target, ok }() if !ok { @@ -395,6 +525,7 @@ func (o *ExtractOptions) extractCommand(command string) error { w = bw var hash hash.Hash + var hashFn = sha256.New closeFn := func() error { return nil } if target.AsArchive { text := strings.Replace(target.Readme, `\u0060`, "`", -1) @@ -535,94 +666,18 @@ func (o *ExtractOptions) extractCommand(command string) error { func() { extractLock.Lock() defer extractLock.Unlock() - delete(targetsByName, layer.Mapping.Name) + delete(tr.targetsByName, layer.Mapping.Name) if hash != nil { - hashByTargetName[layer.Mapping.To] = hex.EncodeToString(hash.Sum(nil)) + tr.hashByTargetName[layer.Mapping.To] = hex.EncodeToString(hash.Sum(nil)) } }() return false, nil } if err := opts.Run(); err != nil { - return err - } - - if willArchive { - buf := &bytes.Buffer{} - fmt.Fprintf(buf, heredoc.Doc(` - Client tools for OpenShift - -------------------------- - - These archives contain the client tooling for [OpenShift](https://docs.openshift.com). - - To verify the contents of this directory, use the 'gpg' and 'shasum' tools to - ensure the archives you have downloaded match those published from this location. - - The openshift-install binary has been preconfigured to install the following release: - - --- - - `)) - if err := describeReleaseInfo(buf, release, false, false, true, false); err != nil { - return err - } - filename := "release.txt" - if err := ioutil.WriteFile(filepath.Join(dir, filename), buf.Bytes(), 0644); err != nil { - return err - } - hash := hashFn() - hash.Write(buf.Bytes()) - hashByTargetName[filename] = hex.EncodeToString(hash.Sum(nil)) - } - - // write a checksum of the tar files to disk as sha256sum.txt.asc - if len(hashByTargetName) > 0 { - var keys []string - for k := range hashByTargetName { - keys = append(keys, k) - } - sort.Strings(keys) - var lines []string - for _, k := range keys { - hash := hashByTargetName[k] - lines = append(lines, fmt.Sprintf("%s %s", hash, filepath.Base(k))) - } - // ensure a trailing newline - if len(lines[len(lines)-1]) != 0 { - lines = append(lines, "") - } - // write the content manifest - data := []byte(strings.Join(lines, "\n")) - filename := "sha256sum.txt" - if err := ioutil.WriteFile(filepath.Join(dir, filename), data, 0644); err != nil { - return fmt.Errorf("unable to write checksum file: %v", err) - } - // sign the content manifest - if signer != nil { - buf := &bytes.Buffer{} - if err := openpgp.ArmoredDetachSign(buf, signer, bytes.NewBuffer(data), nil); err != nil { - return fmt.Errorf("unable to sign the sha256sum.txt file: %v", err) - } - if err := ioutil.WriteFile(filepath.Join(dir, filename+".asc"), buf.Bytes(), 0644); err != nil { - return fmt.Errorf("unable to write signed manifest: %v", err) - } - } - } - - // if we did not process some targets, report that to the user and error if necessary - if len(targetsByName) > 0 { - var missing []string - for _, target := range targetsByName { - missing = append(missing, target.Mapping.From) - } - sort.Strings(missing) - if len(missing) == 1 { - return fmt.Errorf("image did not contain %s", missing[0]) - } - return fmt.Errorf("unable to find multiple files: %s", strings.Join(missing, ", ")) + return targetReleaseInfo{}, err } - - return nil + return tr, nil } const ( diff --git a/pkg/cli/admin/release/info.go b/pkg/cli/admin/release/info.go index ebe5b1fb6c..fce39bbca6 100644 --- a/pkg/cli/admin/release/info.go +++ b/pkg/cli/admin/release/info.go @@ -367,6 +367,9 @@ func (o *InfoOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []st o.From = o.Images[0] o.Images = o.Images[1:] } + if err = o.SecurityOptions.Complete(f, ""); err != nil { + return err + } return nil } @@ -454,10 +457,10 @@ func (o *InfoOptions) Run() error { done := make(chan struct{}) go func() { defer close(done) - baseRelease, baseErr = o.LoadReleaseInfo(o.From, fetchImages) + baseRelease, baseErr = o.LoadReleaseInfo(o.From, fetchImages, false, "") }() - release, err := o.LoadReleaseInfo(o.Images[0], fetchImages) + release, err := o.LoadReleaseInfo(o.Images[0], fetchImages, false, "") if err != nil { return err } @@ -482,7 +485,7 @@ func (o *InfoOptions) Run() error { var exitErr error for _, image := range o.Images { - release, err := o.LoadReleaseInfo(image, fetchImages) + release, err := o.LoadReleaseInfo(image, fetchImages, false, "") if err != nil { exitErr = kcmdutil.ErrExit fmt.Fprintf(o.ErrOut, "error: %v\n", err) @@ -734,16 +737,80 @@ func (i *ReleaseInfo) Platform() string { return fmt.Sprintf("%s/%s", os, arch) } -func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool) (*ReleaseInfo, error) { +func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool, setImageSourcePrefix bool, icspFile string) (*ReleaseInfo, error) { + opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) + opts.SecurityOptions = o.SecurityOptions + opts.FileDir = o.FileDir ref, err := imagesource.ParseReference(image) if err != nil { return nil, err } + fromContext, err := opts.SecurityOptions.Context(ref.Ref) + if err != nil { + return nil, err + } + sourceOpts := &imagesource.Options{ + FileDir: o.FileDir, + Insecure: o.SecurityOptions.Insecure, + RegistryContext: fromContext, + } + if setImageSourcePrefix { + if len(icspFile) > 0 { + opts.SecurityOptions.ImageContentSourcePolicyFile = icspFile + err := opts.SecurityOptions.AddImageSourcePoliciesFromFile(image) + if err != nil { + return nil, err + } + altSources, err := fromContext.AddImageSources(ref.Ref, opts.SecurityOptions.ImageContentSourcePolicyList) + if err != nil { + return nil, err + } + sourceOpts.RegistryContext.ImageSources = altSources + } else { + // now try to look for ICSPs from cluster + // only look for ICSP if release lookup from image reference fails + // such as when working with mirrored registry + if err := opts.SecurityOptions.AddICSPsFromCluster(); err != nil { + return nil, err + } + altSources, err := fromContext.AddImageSources(ref.Ref, opts.SecurityOptions.ImageContentSourcePolicyList) + if err != nil { + return nil, err + } + sourceOpts.RegistryContext.ImageSources = altSources + } + + // Try alternative image sources rather than only the single reference source + // imageSources is a slice of 'registry/repo/name' from ImageContentSourcePolicies + var err error + imageRef, err := imagereference.Parse(image) + if err != nil { + return nil, err + } + for _, icsRef := range sourceOpts.RegistryContext.ImageSources { + icsRef.ID = imageRef.ID + icsRef.Tag = imageRef.Tag + ref, err = imagesource.ParseReference(icsRef.String()) + if err != nil { + return nil, err + } + if _, _, err = sourceOpts.Repository(context.TODO(), ref); err == nil { + image = icsRef.String() + break + } + } + if err != nil { + return nil, err + } + } + + ref, err = imagesource.ParseReference(image) + if err != nil { + return nil, err + } + verifier := imagemanifest.NewVerifier() - opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) - opts.SecurityOptions = o.SecurityOptions - opts.FileDir = o.FileDir release := &ReleaseInfo{ Image: image, @@ -783,6 +850,58 @@ func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool) (*Relea errs = append(errs, err) return true, nil } + if setImageSourcePrefix { + userGivenRef := ref.Ref.AsRepository() + imageSet := false + for _, tag := range is.Spec.Tags { + // If useImageContentSources true, try every one of imageSources rather than it's single reference. + // imagereference.Parse returns the digest ID of each component in the release image-reference. + // If can't get digest ID, skip this tag, this happens when user has built a payload by + // replacing component images in the release with a new image + // imageSources is slice of 'registry/repo/name' determined from ICSP + for _, icsRef := range fromContext.ImageSources { + tagRef, err := imagereference.Parse(tag.From.Name) + // if err != nil, skip this tag + if err == nil { + icsRef.ID = tagRef.ID + srcRef, err := imagesource.ParseReference(icsRef.String()) + if err != nil { + return true, err + } + if _, _, err = sourceOpts.Repository(context.TODO(), srcRef); err == nil { + tag.From.Name = icsRef.String() + imageSet = true + // ignore error, if there's an error don't substitute + break + } + } + } + if !imageSet { + // if user passed the flag to use an iCSP file but no image was set from this ICSP file, error + if len(o.SecurityOptions.ImageContentSourcePolicyFile) > 0 { + return false, fmt.Errorf("could not find image source from ImageContentSourceFile %v", o.SecurityOptions.ImageContentSourcePolicyFile) + } + // Now try the registry/repo passed from the user. If the image does not exist, + // proceed with the release info from the release image-references, from readReleaseImageReference above + tagRef, err := imagereference.Parse(tag.From.Name) + // if err != nil, skip this tag + if err == nil { + userGivenRef.ID = tagRef.ID + userGivenTypedRef, err := imagesource.ParseReference(userGivenRef.String()) + if err != nil { + return true, err + } + // If the user-given registry/repo/name:digest exists, replace with that, if not keep the + // is.Spec.Tag from release image-reference + // if userGivenImage:digestID exists, set that. If not, return original error + if _, _, err = sourceOpts.Repository(context.TODO(), userGivenTypedRef); err == nil { + // ignore error, if there's an error don't substitute + tag.From.Name = userGivenRef.String() + } + } + } + } + } release.References = is case "release-metadata": data, err := ioutil.ReadAll(r) @@ -835,7 +954,7 @@ func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool) (*Relea var lock sync.Mutex release.Images = make(map[string]*Image) r := &imageinfo.ImageRetriever{ - FileDir: opts.FileDir, + FileDir: o.FileDir, Image: make(map[string]imagesource.TypedImageReference), SecurityOptions: o.SecurityOptions, ParallelOptions: o.ParallelOptions, diff --git a/pkg/cli/admin/release/mirror.go b/pkg/cli/admin/release/mirror.go index bfa9c55b3d..4a4dd00689 100644 --- a/pkg/cli/admin/release/mirror.go +++ b/pkg/cli/admin/release/mirror.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "io/ioutil" "net/http" "os" @@ -50,8 +49,9 @@ import ( ) // configFilesBaseDir is created under '--to-dir', when specified, to contain release image -// signature files. It is not used when '--release-image-signature-to-dir` is specified -// which takes precedence over '--to-dir'. +// signature and icsp files. It is not used for signature file when '--release-image-signature-to-dir' is specified. +// It is not used for icsp file when '--release-image-icsp-to-dir' is specified. +// The signature-to and icsp-to flags take precedence over '--to-dir'. const configFilesBaseDir = "config" // maxDigestHashLen is used to truncate digest hash portion before using as part of @@ -61,6 +61,9 @@ const maxDigestHashLen = 16 // signatureFileNameFmt defines format of the release image signature file name const signatureFileNameFmt = "signature-%s-%s.yaml" +// icspFileNameFmt defines format of the release image ImageContentSourcePolicy file name +const icspFileNameFmt = "icsp-%s-%s.yaml" + // NewMirrorOptions creates the options for mirroring a release. func NewMirrorOptions(streams genericclioptions.IOStreams) *MirrorOptions { return &MirrorOptions{ @@ -106,30 +109,36 @@ func NewMirror(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C that can be used to upload the release to another registry. You may use --apply-release-image-signature, --release-image-signature-to-dir, or both - to control the handling of the signature ConfigMap. Option - --apply-release-image-signature will apply the ConfigMap directly to a connected - cluster while --release-image-signature-to-dir specifies an export target directory. If + to control the handling of the signature ConfigMap file. Option + --apply-release-image-signature will apply the release signature ConfigMap directly to a connected + cluster while --release-image-signature-to-dir specifies an export target directory. If --release-image-signature-to-dir is not specified but --to-dir is, --release-image-signature-to-dir defaults to a 'config' subdirectory of --to-dir. The --overwrite option only applies when --apply-release-image-signature is specified and indicates to update an exisiting ConfigMap if one is found. A ConfigMap written to a directory will always replace onethat already exists. + + You may use --release-image-icsp-to-dir to specifiy the export target directory of the + ImageContentSourcePolicy file. The default is 'config'. If not specified but --to-dir is, + --release-image-icsp-to-dir defaults to a 'config' subdirectory of --to-dir. `), Example: templates.Examples(` - # Perform a dry run showing what would be mirrored, including the mirror objects + # Perform a dry run showing what would be mirrored, including the mirror objects, and control where signature and ICSP files are written oc adm release mirror 4.3.0 --to myregistry.local/openshift/release \ - --release-image-signature-to-dir /tmp/releases --dry-run + --release-image-signature-to-dir /tmp/releases \ + --release-image-icsp-to-dir /tmp/icsps --dry-run - # Mirror a release into the current directory + # Mirror a release into the current directory and control where signature and ICSP files are written oc adm release mirror 4.3.0 --to file://openshift/release \ - --release-image-signature-to-dir /tmp/releases + --release-image-signature-to-dir /tmp/releases \ + --release-image-icsp-to-dir /tmp/icsps # Mirror a release to another directory in the default location oc adm release mirror 4.3.0 --to-dir /tmp/releases # Upload a release from the current directory to another server oc adm release mirror --from file://openshift/release --to myregistry.com/openshift/release \ - --release-image-signature-to-dir /tmp/releases + --release-image-config-dir /tmp/releases # Mirror the 4.3.0 release to repository registry.example.com and apply signatures to connected cluster oc adm release mirror --from=quay.io/openshift-release-dev/ocp-release:4.3.0-x86_64 \ @@ -150,10 +159,11 @@ func NewMirror(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C flags.StringVar(&o.ToImageStream, "to-image-stream", o.ToImageStream, "An image stream to tag images into.") flags.StringVar(&o.FromDir, "from-dir", o.FromDir, "A directory to import images from.") flags.StringVar(&o.ToDir, "to-dir", o.ToDir, "A directory to export images to.") + flags.StringVar(&o.ReleaseImageICSPToDir, "release-image-icsp-to-dir", o.ReleaseImageICSPToDir, "Path to write ImageContentSourcePolicy file. If not set, defaults to 'config'.") + flags.StringVar(&o.ReleaseImageSignatureToDir, "release-image-signature-to-dir", o.ReleaseImageSignatureToDir, "Path to write release image signature ConfigMap files. If not set, defaults to 'config'.") flags.BoolVar(&o.ToMirror, "to-mirror", o.ToMirror, "Output the mirror mappings instead of mirroring.") flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Display information about the mirror without actually executing it.") flags.BoolVar(&o.ApplyReleaseImageSignature, "apply-release-image-signature", o.ApplyReleaseImageSignature, "Apply release image signature to connected cluster.") - flags.StringVar(&o.ReleaseImageSignatureToDir, "release-image-signature-to-dir", o.ReleaseImageSignatureToDir, "A directory to export release image signature to.") flags.BoolVar(&o.SkipRelease, "skip-release-image", o.SkipRelease, "Do not push the release image.") flags.StringVar(&o.ToRelease, "to-release-image", o.ToRelease, "Specify an alternate locations for the release image instead as tag 'release' in --to.") @@ -182,6 +192,7 @@ type MirrorOptions struct { ApplyReleaseImageSignature bool ReleaseImageSignatureToDir string + ReleaseImageICSPToDir string Overwrite bool DryRun bool @@ -215,6 +226,10 @@ func (o *MirrorOptions) Complete(cmd *cobra.Command, f kcmdutil.Factory, args [] } o.From = args[0] + if err := o.SecurityOptions.Complete(f, ""); err != nil { + return err + } + o.ImageClientFn = func() (imageclient.Interface, string, error) { cfg, err := f.ToRESTConfig() if err != nil { @@ -242,7 +257,6 @@ func (o *MirrorOptions) Complete(cmd *cobra.Command, f kcmdutil.Factory, args [] client := coreClient.ConfigMaps(configmap.NamespaceLabelConfigMap) return client, nil } - o.PrintImageContentInstructions = true return nil } @@ -280,6 +294,10 @@ func (o *MirrorOptions) Validate() error { o.ReleaseImageSignatureToDir = filepath.Join(o.ToDir, configFilesBaseDir) } + if len(o.ReleaseImageICSPToDir) == 0 && len(o.ToDir) > 0 { + o.ReleaseImageICSPToDir = filepath.Join(o.ToDir, configFilesBaseDir) + } + if o.Overwrite && !o.ApplyReleaseImageSignature { return fmt.Errorf("--overwite is only valid when --apply-release-image-signature is specified") } @@ -312,6 +330,19 @@ func createSignatureFileName(digest string) (string, error) { return fmt.Sprintf(signatureFileNameFmt, algo, hash), nil } +func createICSPFileName(digest string) (string, error) { + parts := strings.SplitN(digest, ":", 3) + if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 { + return "", fmt.Errorf("the provided digest, %s, must be of the form ALGO:HASH", digest) + } + algo, hash := parts[0], parts[1] + + if len(hash) > maxDigestHashLen { + hash = hash[:maxDigestHashLen] + } + return fmt.Sprintf(icspFileNameFmt, algo, hash), nil +} + // handleSignatures implements the image release signature configmap specific logic. // Signature configmaps may be written to a directory or applied to a cluster. func (o *MirrorOptions) handleSignatures(context context.Context, signaturesByDigest map[string][][]byte) error { @@ -520,10 +551,11 @@ func (o *MirrorOptions) Run() error { if err := imageVerifier.Verify(ctx, releaseDigest); err != nil { fmt.Fprintf(o.ErrOut, "warning: An image was retrieved that failed verification: %v\n", err) } + var srcRef imagesource.TypedImageReference var mappings []mirror.Mapping if len(o.From) > 0 { src := o.From - srcRef, err := imagesource.ParseReference(src) + srcRef, err = imagesource.ParseReference(src) if err != nil { return fmt.Errorf("invalid --from: %v", err) } @@ -531,16 +563,11 @@ func (o *MirrorOptions) Run() error { // if the source ref is a file type, provide a function that checks the local file store for a given manifest // before continuing, to allow mirroring an entire release to disk in a single file://REPO. if srcRef.Type == imagesource.DestinationFile { - if repo, err := (&imagesource.Options{FileDir: o.FromDir}).Repository(context.TODO(), srcRef); err == nil { + if _, manifests, err := (&imagesource.Options{FileDir: o.FromDir}).Repository(context.TODO(), srcRef); err == nil { sourceFn = func(ref imagesource.TypedImageReference) imagesource.TypedImageReference { if ref.Type == imagesource.DestinationFile || len(ref.Ref.ID) == 0 { return ref } - manifests, err := repo.Manifests(context.TODO()) - if err != nil { - klog.V(2).Infof("Unable to get local manifest service: %v", err) - return ref - } ok, err := manifests.Exists(context.TODO(), digest.Digest(ref.Ref.ID)) if err != nil { klog.V(2).Infof("Unable to get check for local manifest: %v", err) @@ -583,6 +610,7 @@ func (o *MirrorOptions) Run() error { repositories := make(map[string]struct{}) // build the mapping list for mirroring and rewrite if necessary + //userGivenRef := srcRef.Ref.AsRepository() for i := range is.Spec.Tags { tag := &is.Spec.Tags[i] if tag.From == nil || tag.From.Kind != "DockerImage" { @@ -596,6 +624,10 @@ func (o *MirrorOptions) Run() error { return fmt.Errorf("image-references should only contain pointers to images by digest: %s", tag.From.Name) } + opts := mirror.NewMirrorImageOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) + opts.SecurityOptions = o.SecurityOptions + opts.ParallelOptions = o.ParallelOptions + opts.FileDir = o.ToDir // Allow mirror refs to be sourced locally srcMirrorRef := imagesource.TypedImageReference{Ref: from, Type: imagesource.DestinationRegistry} srcMirrorRef = sourceFn(srcMirrorRef) @@ -786,6 +818,12 @@ func (o *MirrorOptions) Run() error { fmt.Fprintf(o.Out, "Mirrored to: %s\n", t) } } + if len(o.ReleaseImageICSPToDir) == 0 { + o.ReleaseImageICSPToDir = configFilesBaseDir + } + if err := o.printImageContentInstructions(repositories, toList, releaseDigest); err != nil { + return fmt.Errorf("Error creating mirror usage instructions: %v", err) + } } if toDisk { if len(o.ToDir) > 0 { @@ -793,12 +831,6 @@ func (o *MirrorOptions) Run() error { } else { fmt.Fprintf(o.Out, "\nTo upload local images to a registry, run:\n\n oc image mirror 'file://%s*' REGISTRY/REPOSITORY\n\n", to) } - } else if len(toList) > 0 { - if o.PrintImageContentInstructions { - if err := printImageContentInstructions(o.Out, o.From, toList, o.ReleaseImageSignatureToDir, repositories); err != nil { - return fmt.Errorf("Error creating mirror usage instructions: %v", err) - } - } } if o.ApplyReleaseImageSignature || len(o.ReleaseImageSignatureToDir) > 0 { signatures := imageVerifier.Signatures() @@ -824,7 +856,7 @@ func (o *MirrorOptions) Run() error { // printImageContentInstructions provides examples to the user for using the new repository mirror // https://github.com/openshift/installer/blob/master/docs/dev/alternative_release_image_sources.md -func printImageContentInstructions(out io.Writer, from string, toList []string, signatureToDir string, repositories map[string]struct{}) error { +func (o *MirrorOptions) printImageContentInstructions(repositories map[string]struct{}, toList []string, digest string) error { type installConfigSubsection struct { ImageContentSources []operatorv1alpha1.RepositoryDigestMirrors `json:"imageContentSources"` } @@ -835,17 +867,16 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, mirrorRef, err := imagesource.ParseReference(to) if err != nil { return fmt.Errorf("Unable to parse image reference '%s': %v", to, err) - } - if mirrorRef.Type != imagesource.DestinationRegistry { - return nil - } - mirrorRepo := mirrorRef.Ref.AsRepository().String() + if mirrorRef.Type != imagesource.DestinationRegistry { + return nil + } + mirrorRepo := mirrorRef.Ref.AsRepository().String() - if len(from) != 0 { - sourceRef, err := imagesource.ParseReference(from) - if err != nil { - return fmt.Errorf("Unable to parse image reference '%s': %v", from, err) - } + if len(o.From) != 0 { + sourceRef, err := imagesource.ParseReference(o.From) + if err != nil { + return fmt.Errorf("Unable to parse image reference '%s': %v", o.From, err) + } if sourceRef.Type != imagesource.DestinationRegistry { return nil } @@ -867,6 +898,7 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, return sources[i].Source < sources[j].Source }) } +} // Create and display install-config.yaml example imageContentSources := installConfigSubsection{ @@ -875,16 +907,19 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, if err != nil { return fmt.Errorf("Unable to marshal install-config.yaml example yaml: %v", err) } - fmt.Fprintf(out, "\nTo use the new mirrored repository to install, add the following section to the install-config.yaml:\n\n") - fmt.Fprintf(out, string(installConfigExample)) + fmt.Fprintf(o.Out, "\nTo use the new mirrored repository to install, add the following section to the install-config.yaml:\n\n") + fmt.Fprintf(o.Out, string(installConfigExample)) + + // Create and display ImageContentSourcePolicy + mirrorRepoStripped := strings.FieldsFunc(mirrorRef.Ref.Name, func(r rune) bool { return strings.ContainsRune(" .:/", r) }) + icspName := strings.Join(mirrorRepoStripped[:], "-") - // Create and display ImageContentSourcePolicy example icsp := operatorv1alpha1.ImageContentSourcePolicy{ TypeMeta: metav1.TypeMeta{ APIVersion: operatorv1alpha1.GroupVersion.String(), Kind: "ImageContentSourcePolicy"}, ObjectMeta: metav1.ObjectMeta{ - Name: "example", + Name: icspName, }, Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ RepositoryDigestMirrors: sources, @@ -899,16 +934,30 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, } delete(unstructuredObj.Object["metadata"].(map[string]interface{}), "creationTimestamp") - icspExample, err := yaml.Marshal(unstructuredObj.Object) + icspDataBytes, err := yaml.Marshal(unstructuredObj.Object) if err != nil { - return fmt.Errorf("Unable to marshal ImageContentSourcePolicy example yaml: %v", err) + return fmt.Errorf("Unable to marshal ImageContentSourcePolicy yaml: %v", err) } - fmt.Fprintf(out, "\n\nTo use the new mirrored repository for upgrades, use the following to create an ImageContentSourcePolicy:\n\n") - fmt.Fprintf(out, string(icspExample)) - - if len(signatureToDir) != 0 { - fmt.Fprintf(out, "\n\nTo apply signature configmaps use 'oc apply' on files found in %s\n\n", signatureToDir) + icspFileName, err := createICSPFileName(digest) + if err != nil { + return fmt.Errorf("creating filename: %v", err) } + icspFullName := filepath.Join(o.ReleaseImageICSPToDir, icspFileName) + if o.DryRun { + fmt.Fprintf(o.Out, "\ninfo: Write ImageContentSourcePolicy file %s\n", icspFullName) + } else { + if err := os.MkdirAll(filepath.Dir(icspFullName), 0750); err != nil { + return err + } + if err := ioutil.WriteFile(icspFullName, icspDataBytes, 0640); err != nil { + return err + } + fmt.Fprintf(o.Out, "\nImageContentSourcePolicy file %s created\n", icspFullName) + } + fmt.Fprintf(o.Out, "\n\nTo use the new mirrored repository for upgrades, use the following to create an ImageContentSourcePolicy:\n\n") + fmt.Fprintf(o.Out, string(icspDataBytes)) + + fmt.Fprintf(o.Out, "\n\nTo apply ImageContentSourcePolicy, use 'oc apply' on file %s\n\n", icspFullName) return nil } diff --git a/pkg/cli/image/append/append.go b/pkg/cli/image/append/append.go index 1916f3d0b9..087ff11426 100644 --- a/pkg/cli/image/append/append.go +++ b/pkg/cli/image/append/append.go @@ -210,7 +210,7 @@ func (o *AppendImageOptions) Run() error { } ctx := context.Background() - fromContext, err := o.SecurityOptions.Context() + fromContext, err := o.SecurityOptions.Context(from.Ref) if err != nil { return err } @@ -230,30 +230,25 @@ func (o *AppendImageOptions) Run() error { RegistryContext: toContext, } - toRepo, err := toOptions.Repository(ctx, to) + toRepo, toManifests, err := toOptions.Repository(ctx, to) if err != nil { return err } - toManifests, err := toRepo.Manifests(ctx) - if err != nil { - return err - } - var ( base *dockerv1client.DockerImageConfig baseDigest digest.Digest baseContentDigest digest.Digest layers []distribution.Descriptor fromRepo distribution.Repository + srcManifest distribution.Manifest + manifestLocation imagemanifest.ManifestLocation ) if from != nil { - repo, err := fromOptions.Repository(ctx, *from) + repo, _, err := toOptions.Repository(ctx, *from) if err != nil { return err } - fromRepo = repo - - srcManifest, manifestLocation, err := imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) + srcManifest, manifestLocation, err = imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) if err != nil { return fmt.Errorf("unable to read image %s: %v", from, err) } @@ -261,7 +256,6 @@ func (o *AppendImageOptions) Run() error { if err != nil { return fmt.Errorf("unable to parse image %s: %v", from, err) } - contentDigest, err := registryclient.ContentDigestForManifest(srcManifest, manifestLocation.Manifest.Algorithm()) if err != nil { return err @@ -269,7 +263,7 @@ func (o *AppendImageOptions) Run() error { baseDigest = manifestLocation.Manifest baseContentDigest = contentDigest - + fromRepo = repo } else { base = add.NewEmptyConfig() layers = nil diff --git a/pkg/cli/image/extract/extract.go b/pkg/cli/image/extract/extract.go index 4f9d36c8d2..4478737567 100644 --- a/pkg/cli/image/extract/extract.go +++ b/pkg/cli/image/extract/extract.go @@ -135,7 +135,7 @@ func NewExtractOptions(streams genericclioptions.IOStreams) *ExtractOptions { } // New creates a new command -func NewExtract(streams genericclioptions.IOStreams) *cobra.Command { +func NewExtract(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewExtractOptions(streams) cmd := &cobra.Command{ @@ -144,7 +144,7 @@ func NewExtract(streams genericclioptions.IOStreams) *cobra.Command { Long: desc, Example: example, Run: func(c *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(c, args)) + kcmdutil.CheckErr(o.Complete(f, c, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -286,11 +286,10 @@ func parseMappings(images, paths, files []string, requireEmpty bool) ([]Mapping, return mappings, nil } -func (o *ExtractOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *ExtractOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { return err } - if len(args) == 0 { return fmt.Errorf("you must specify at least one image to extract as an argument") } @@ -304,6 +303,11 @@ func (o *ExtractOptions) Complete(cmd *cobra.Command, args []string) error { if err != nil { return err } + + if err := o.SecurityOptions.Complete(f, ""); err != nil { + return err + } + return nil } @@ -316,7 +320,8 @@ func (o *ExtractOptions) Validate() error { func (o *ExtractOptions) Run() error { ctx := context.Background() - fromContext, err := o.SecurityOptions.Context() + var err error + fromContext, err := o.SecurityOptions.Context(o.Mappings[0].ImageRef.Ref) if err != nil { return err } @@ -325,7 +330,6 @@ func (o *ExtractOptions) Run() error { Insecure: o.SecurityOptions.Insecure, RegistryContext: fromContext, } - stopCh := make(chan struct{}) defer close(stopCh) q := workqueue.New(o.ParallelOptions.MaxPerRegistry, stopCh) @@ -334,9 +338,9 @@ func (o *ExtractOptions) Run() error { mapping := o.Mappings[i] from := mapping.ImageRef q.Try(func() error { - repo, err := fromOptions.Repository(ctx, from) - if err != nil { - return fmt.Errorf("unable to connect to image repository %s: %v", from.String(), err) + repo, _, err := fromOptions.Repository(ctx, from) + if err != nil || repo == nil { + return fmt.Errorf("unable to connect to image repository %s: %v", from, err) } srcManifest, location, err := imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) @@ -351,17 +355,14 @@ func (o *ExtractOptions) Run() error { } return fmt.Errorf("unable to read image %s: %v", from, err) } - contentDigest, err := registryclient.ContentDigestForManifest(srcManifest, location.Manifest.Algorithm()) if err != nil { return err } - imageConfig, layers, err := imagemanifest.ManifestToImageConfig(ctx, srcManifest, repo.Blobs(ctx), location) if err != nil { - return fmt.Errorf("unable to parse image %s: %v", from, err) + return err } - if mapping.ConditionFn != nil { ok, err := mapping.ConditionFn(&mapping, location.Manifest, imageConfig) if err != nil { diff --git a/pkg/cli/image/image.go b/pkg/cli/image/image.go index 48d9a8351b..ded56d30da 100644 --- a/pkg/cli/image/image.go +++ b/pkg/cli/image/image.go @@ -44,7 +44,7 @@ func NewCmdImage(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra Commands: []*cobra.Command{ serve.NewServe(streams), append.NewCmdAppendImage(streams), - extract.NewExtract(streams), + extract.NewExtract(f, streams), }, }, } diff --git a/pkg/cli/image/imagesource/options.go b/pkg/cli/image/imagesource/options.go index 4662b0e680..e01e5bd635 100644 --- a/pkg/cli/image/imagesource/options.go +++ b/pkg/cli/image/imagesource/options.go @@ -3,6 +3,7 @@ package imagesource import ( "context" "fmt" + "net/url" "github.com/docker/distribution" "github.com/openshift/library-go/pkg/image/registryclient" @@ -17,26 +18,73 @@ type Options struct { RegistryContext *registryclient.Context } -// Repository retrieves the appropriate repository implementation for the given typed reference. -func (o *Options) Repository(ctx context.Context, ref TypedImageReference) (distribution.Repository, error) { +// Repository retrieves the appropriate repository implementation and ManifestService for the given typed reference. +func (o *Options) Repository(ctx context.Context, ref TypedImageReference) (distribution.Repository, distribution.ManifestService, error) { + o.RegistryContext.RepositoryRetriever = ®istryContext{o.RegistryContext} switch ref.Type { case DestinationRegistry: - return o.RegistryContext.Repository(ctx, ref.Ref.DockerClientDefaults().RegistryURL(), ref.Ref.RepositoryName(), o.Insecure) + repo, err := o.RegistryContext.Repository(ctx, ref.Ref.DockerClientDefaults().RegistryURL(), ref.Ref.RepositoryName(), o.Insecure) + if err != nil { + return nil, nil, err + } + manifests, err := repo.Manifests(context.TODO()) + if err != nil { + return nil, nil, fmt.Errorf("unable to get local manifest service: %v", err) + } + return repo, manifests, nil case DestinationFile: driver := &fileDriver{ BaseDir: o.FileDir, } - return driver.Repository(ctx, ref.Ref.DockerClientDefaults().RegistryURL(), ref.Ref.RepositoryName(), o.Insecure) + repo, err := driver.Repository(ctx, ref.Ref.DockerClientDefaults().RegistryURL(), ref.Ref.RepositoryName(), o.Insecure) + if err != nil { + return nil, nil, err + } + return repo, nil, nil case DestinationS3: driver := &s3Driver{ Creds: o.RegistryContext.Credentials, CopyFrom: o.AttemptS3BucketCopy, } url := ref.Ref.DockerClientDefaults().RegistryURL() - return driver.Repository(ctx, url, ref.Ref.RepositoryName(), o.Insecure) + repo, err := driver.Repository(ctx, url, ref.Ref.RepositoryName(), o.Insecure) + if err != nil { + return nil, nil, err + } + return repo, nil, nil default: - return nil, fmt.Errorf("unrecognized image reference type %s", ref.Type) + return nil, nil, fmt.Errorf("unrecognized image reference type %s", ref.Type) + } +} + +type registryContext struct { + *registryclient.Context +} + +func (c *registryContext) Repository(ctx context.Context, registry *url.URL, repoName string, insecure bool) (distribution.Repository, error) { + var repo distribution.Repository + var err error + if len(c.ImageSources) == 0 { + repo, err = c.Repository(ctx, registry, repoName, insecure) + if err != nil { + return nil, err + } + } + for _, ics := range c.ImageSources { + repo, err = c.Repository(ctx, ics.RegistryURL(), ics.RepositoryName(), insecure) + if err != nil { + continue + } + + // it would be nice to simply return ManifestService here, as we'll need it, but this will not satifsy library-go's RepositoryRetriever interface + _, err := repo.Manifests(context.TODO()) + if err != nil { + err = fmt.Errorf("unable to get local manifest service: %v", err) + continue + } + break } + return repo, nil } // ExpandWildcard expands the provided typed reference (which is known to have an expansion) @@ -48,7 +96,7 @@ func (o *Options) ExpandWildcard(ref TypedImageReference) ([]TypedImageReference } // lookup tags that match the search - repo, err := o.Repository(context.Background(), ref) + repo, _, err := o.Repository(context.Background(), ref) if err != nil { return nil, err } diff --git a/pkg/cli/image/info/info.go b/pkg/cli/image/info/info.go index be662ad6bb..ed7d0776f5 100644 --- a/pkg/cli/image/info/info.go +++ b/pkg/cli/image/info/info.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/openshift/library-go/pkg/image/dockerv1client" + imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" "github.com/openshift/oc/pkg/cli/image/imagesource" imagemanifest "github.com/openshift/oc/pkg/cli/image/manifest" @@ -109,16 +110,15 @@ func (o *InfoOptions) Run() error { if len(o.Images) == 0 { return fmt.Errorf("must specify one or more images as arguments") } - - // cache the context - context, err := o.SecurityOptions.Context() + regContext, err := o.SecurityOptions.Context(imagereference.DockerImageReference{}) if err != nil { return err } + opts := &imagesource.Options{ FileDir: o.FileDir, Insecure: o.SecurityOptions.Insecure, - RegistryContext: context, + RegistryContext: regContext, } hadError := false @@ -364,7 +364,7 @@ type ImageRetriever struct { func (o *ImageRetriever) Run() error { ctx := context.Background() - fromContext, err := o.SecurityOptions.Context() + fromContext, err := o.SecurityOptions.Context(imagereference.DockerImageReference{}) if err != nil { return err } @@ -388,12 +388,12 @@ func (o *ImageRetriever) Run() error { name := key from := o.Image[key] q.Try(func() error { - repo, err := fromOptions.Repository(ctx, from) + repo, manifests, err := fromOptions.Repository(ctx, from) if err != nil { return callbackFn(name, nil, fmt.Errorf("unable to connect to image repository %s: %v", from, err)) } - allManifests, manifestList, listDigest, err := imagemanifest.AllManifests(ctx, from.Ref, repo) + allManifests, manifestList, listDigest, err := imagemanifest.AllManifests(ctx, from.Ref, manifests, repo) if err != nil { if imagemanifest.IsImageForbidden(err) { msg := fmt.Sprintf("image %q does not exist or you don't have permission to access the repository", from) diff --git a/pkg/cli/image/manifest/.security_test.go.swp b/pkg/cli/image/manifest/.security_test.go.swp new file mode 100644 index 0000000000000000000000000000000000000000..748383ff318e36bfdb81542fb7ad8f3b6e6b71e7 GIT binary patch literal 16384 zcmeI3Uu+yl9ml6>2wa;&pn?|!Os=##C-$ygNE;mlDYD~Y(X^p<(^ge+*L!p4jrVq! z-8rX+V-gS#NW7rJ3lb7kDr!}s(1%tPR4NdaKT<(z#lIjyAByM;h^Po65fJeG&7Zx! z^Id93sR-4s^zrTO&dkny=QlII-`sj>^61=QzTdf-;n>C4>1R(b&beP;KdaorvbgPq zq3F`BiydL?d*O1NNk^SEkS&OYWcaKt(Dmbna9i>5I9+kG)OTBfT<$D6e%yBBL`2zg z(3EcMyU9w+^~1mo+FnbzZ7&L%B9m?={7xFk?xMsurxg!*R(YvZpj4osKu0zw_Fu<7 zbo10CUA>{Yn_vH~y9)`*(^7#_fl`4|fl`4|fl`4|fl`4|fj7GXGTFvXqLJH7L)Xmb zRRf<_o9~}ApC<Q6%z~ZZpVu(Xc;2cHACfrH?EU}vMUT_sS|5nES4E_MV z1HKB5fkkjPxE;)Z1K?J0J^0rS=mow9V$cL11-rp(Z(;0J@H_Ae@B(-qJO>^D$H0HK zGj;)-2WP-%zyf$L_&qjf&VpZpZ-EEEtzaAY3HtR6cn*9GEQ0&MKJb1({r+yye^lsL zA7|>ba);LzbzV(HD7;Km&E-3njtO5L7SgNpHvXJ8ysi_(ZsKK`Xy{i;s)ej|K5z`pQU9>B zPb)y}nhioIQ#|a|njBL)^;%y{wG}0F6}1+{GU&V-#YPb&QWMqLqAGYLN2H>K{zoy@KN!(||a)TtVVw%Al;xTo#42TuE$OTpj0CFN)=|NOKkQ?}cQW zlC#|jWsrozC8Nn+&(f+|X(V}u)3$mC=EWfou@_ZiEnnK~{lUdS+YH?`D^*{DzPq$O z_ubt>6Yv0PKwozHVjn;YM8@j}bbUL%zXyDeypb%;xokQdwG;38%#Bs`xrvdVE8FMN|<>9oh)%lI9 zn6r@x8G|OB@ZAS^6ohz4g}xo&LEMo+=-h)hOa(3QwSgy&qa{mSsjrWX^T{#xAX6rI{8X>h}s)C-bIJHh*gTW{sX-3jaM)GuDFG1RSSS%F@rRqexJUK^-~p>dO9%Yy?)+hP*)} z;EvMU-?)&ulnCZih2>DZkcuXVe8HIVC_?3d!)7v}ltF(1ub3nL z7*U$ONrl;L^h!F?XF|HG+T;;W$Zmqac@k^vL!LI`IcVw7?8P>=;uoX}E?ZgFD z#wyC7F!56_D;A(II zvHgqSY49a*3ao-Q5a3hbUhomH9sB@s{kAoQ0!FKQ(V*2lbr@$$20<3{W@ImlS@G@ff--2I(C&7K-Hn0oq1ZSb& z0yqK=1JZL6lt-yRsX(c~n?`~4(TDct$VDpn;|Cj!;Z8np#S-53jjQAQxQEr3P8vb8 zsjk6K^Fp&Dn8Bqp`+0YXCca_ab7qjTHY&*&91HU9cbnzVnGqs61h+h_^_SEufPxIf z!~J!)TRPl02*&55r~xqs)jf|THU<-wUg5b<>N$`@xrR2-QO*gPHO}F5k-s%^OO$*t zYL`dO?M>b&S+g~2gldTTQ|JohQL!a&rg2+^Lql6Kq$w&g!eeb6`&*&5Yy*+aqF5C$ zN$IF|nQjP0RHI8K3?1ViJdMrn` z8wYV%PYrEf&%pA|Qf1{uIY;makyndq(S#8!_KcJICG{eBSGN78lMk7~m5Hr-X`eYG zm>hV}Ozf|SbdEe-7=~_$)ynwo6l74)ip#c3XwzU?`K7ZFRHO*IHpXk#A+`Rfw&amU zHiw<~wCVXmsSu}nSeR!}197VU^-_ffrc+fUqZ(~{*jBd}mT#qlHpJ!H?N_6OLUdGP zKr-DoR(AA_=^7~uor170qb^Y6gCeD&+bV?(PQ5#(9@QUeA(+bdI`bgkM9iDqwzUyp zf59erG&@2zX0xBP+fUtWw}#C2ovPq~sTErBI{SK?YNXz{(7z9eH-2ZjZ)@Z-ZIJdi z!EW>RwnazmGn0^mW5^kimR1lPmv2w{f3*{-ek##|>7SQGc1b_E_4T9q1y38%=6-m) q#Ia>|we9A%{VeQu=a;aOsF^@@+`txie}#L=E@@kSpewH!yZbLXwdwW% literal 0 HcmV?d00001 diff --git a/pkg/cli/image/manifest/manifest.go b/pkg/cli/image/manifest/manifest.go index 1f6106a7a6..7507aefac6 100644 --- a/pkg/cli/image/manifest/manifest.go +++ b/pkg/cli/image/manifest/manifest.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "net/http" "regexp" "runtime" "sync" @@ -21,13 +20,11 @@ import ( "github.com/docker/libtrust" "github.com/opencontainers/go-digest" - "k8s.io/client-go/rest" "k8s.io/klog/v2" "github.com/openshift/library-go/pkg/image/dockerv1client" imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" - "github.com/openshift/oc/pkg/cli/image/manifest/dockercredentials" "github.com/openshift/oc/pkg/helpers/image/dockerlayer/add" ) @@ -39,37 +36,6 @@ func (o *ParallelOptions) Bind(flags *pflag.FlagSet) { flags.IntVar(&o.MaxPerRegistry, "max-per-registry", o.MaxPerRegistry, "Number of concurrent requests allowed per registry.") } -type SecurityOptions struct { - RegistryConfig string - Insecure bool - SkipVerification bool - - CachedContext *registryclient.Context -} - -func (o *SecurityOptions) Bind(flags *pflag.FlagSet) { - flags.StringVarP(&o.RegistryConfig, "registry-config", "a", o.RegistryConfig, "Path to your registry credentials (defaults to ~/.docker/config.json)") - flags.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP") - flags.BoolVar(&o.SkipVerification, "skip-verification", o.SkipVerification, "Skip verifying the integrity of the retrieved content. This is not recommended, but may be necessary when importing images from older image registries. Only bypass verification if the registry is known to be trustworthy.") -} - -// ReferentialHTTPClient returns an http.Client that is appropriate for accessing -// blobs referenced outside of the registry (due to the present of the URLs attribute -// in the manifest reference for a layer). -func (o *SecurityOptions) ReferentialHTTPClient() (*http.Client, error) { - ctx, err := o.Context() - if err != nil { - return nil, err - } - client := &http.Client{} - if o.Insecure { - client.Transport = ctx.InsecureTransport - } else { - client.Transport = ctx.Transport - } - return client, nil -} - type Verifier interface { Verify(dgst, contentDgst digest.Digest) Verified() bool @@ -99,39 +65,6 @@ func (v *verifier) Verified() bool { return !v.hadError } -func (o *SecurityOptions) Context() (*registryclient.Context, error) { - if o.CachedContext != nil { - return o.CachedContext, nil - } - context, err := o.NewContext() - if err == nil { - o.CachedContext = context - o.CachedContext.Retries = 3 - } - return context, err -} - -func (o *SecurityOptions) NewContext() (*registryclient.Context, error) { - rt, err := rest.TransportFor(&rest.Config{}) - if err != nil { - return nil, err - } - insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}}) - if err != nil { - return nil, err - } - creds := dockercredentials.NewLocal() - if len(o.RegistryConfig) > 0 { - creds, err = dockercredentials.NewFromFile(o.RegistryConfig) - if err != nil { - return nil, fmt.Errorf("unable to load --registry-config: %v", err) - } - } - context := registryclient.NewContext(rt, insecureRT).WithCredentials(creds) - context.DisableDigestVerification = o.SkipVerification - return context, nil -} - // FilterOptions assist in filtering out unneeded manifests from ManifestList objects. type FilterOptions struct { FilterByOS string @@ -214,7 +147,7 @@ var PreferManifestList = distribution.WithManifestMediaTypes([]string{ }) // AllManifests returns all non-list manifests, the list manifest (if any), the digest the from refers to, or an error. -func AllManifests(ctx context.Context, from imagereference.DockerImageReference, repo distribution.Repository) (map[digest.Digest]distribution.Manifest, *manifestlist.DeserializedManifestList, digest.Digest, error) { +func AllManifests(ctx context.Context, from imagereference.DockerImageReference, manifests distribution.ManifestService, repo distribution.Repository) (map[digest.Digest]distribution.Manifest, *manifestlist.DeserializedManifestList, digest.Digest, error) { var srcDigest digest.Digest if len(from.ID) > 0 { srcDigest = digest.Digest(from.ID) @@ -227,15 +160,10 @@ func AllManifests(ctx context.Context, from imagereference.DockerImageReference, } else { return nil, nil, "", fmt.Errorf("no tag or digest specified") } - manifests, err := repo.Manifests(ctx) - if err != nil { - return nil, nil, "", err - } srcManifest, err := manifests.Get(ctx, srcDigest, PreferManifestList) if err != nil { return nil, nil, "", err } - return ManifestsFromList(ctx, srcDigest, srcManifest, manifests, from) } diff --git a/pkg/cli/image/manifest/security.go b/pkg/cli/image/manifest/security.go new file mode 100644 index 0000000000..e7f3d08d01 --- /dev/null +++ b/pkg/cli/image/manifest/security.go @@ -0,0 +1,222 @@ +package manifest + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + + "github.com/spf13/pflag" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + kcmdutil "k8s.io/kubectl/pkg/cmd/util" + + operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1" + operatorv1alpha1scheme "github.com/openshift/client-go/operator/clientset/versioned/scheme" + operatorv1alpha1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1alpha1" + imagereference "github.com/openshift/library-go/pkg/image/reference" + "github.com/openshift/library-go/pkg/image/registryclient" + "github.com/openshift/oc/pkg/cli/image/manifest/dockercredentials" +) + +type SecurityOptions struct { + RegistryConfig string + Insecure bool + SkipVerification bool + ImageContentSourcePolicyFile string + ImageContentSourcePolicyList []operatorv1alpha1.ImageContentSourcePolicy + ICSPClientFn func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) + + CachedContext *registryclient.Context +} + +func (o *SecurityOptions) Bind(flags *pflag.FlagSet) { + flags.StringVarP(&o.RegistryConfig, "registry-config", "a", o.RegistryConfig, "Path to your registry credentials (defaults to ~/.docker/config.json)") + flags.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP") + flags.BoolVar(&o.SkipVerification, "skip-verification", o.SkipVerification, "Skip verifying the integrity of the retrieved content. This is not recommended, but may be necessary when importing images from older image registries. Only bypass verification if the registry is known to be trustworthy.") + flags.StringVar(&o.ImageContentSourcePolicyFile, "icsp-file", o.ImageContentSourcePolicyFile, "Path to an ImageContentSourcePolicy file. If set, data from this file will be used to set source release image.") +} + +// ReferentialHTTPClient returns an http.Client that is appropriate for accessing +// blobs referenced outside of the registry (due to the present of the URLs attribute +// in the manifest reference for a layer). +func (o *SecurityOptions) ReferentialHTTPClient() (*http.Client, error) { + ctx, err := o.Context(imagereference.DockerImageReference{}) + if err != nil { + return nil, err + } + client := &http.Client{} + if o.Insecure { + client.Transport = ctx.InsecureTransport + } else { + client.Transport = ctx.Transport + } + return client, nil +} + +func (o *SecurityOptions) AddImageSourcePoliciesFromFile(image string) error { + if len(image) == 0 { + return fmt.Errorf("expected image to find image sources") + } + icspData, err := ioutil.ReadFile(o.ImageContentSourcePolicyFile) + if err != nil { + return fmt.Errorf("unable to read ImageContentSourceFile %s: %v", o.ImageContentSourcePolicyFile, err) + } + if len(icspData) == 0 { + return fmt.Errorf("no data found in ImageContentSourceFile %s", o.ImageContentSourcePolicyFile) + } + icspObj, err := runtime.Decode(operatorv1alpha1scheme.Codecs.UniversalDeserializer(), icspData) + if err != nil { + return fmt.Errorf("error decoding ImageContentSourcePolicy from %s: %v", o.ImageContentSourcePolicyFile, err) + } + var icsp *operatorv1alpha1.ImageContentSourcePolicy + var ok bool + if icsp, ok = icspObj.(*operatorv1alpha1.ImageContentSourcePolicy); !ok { + return fmt.Errorf("could not decode ImageContentSourcePolicy from %s", o.ImageContentSourcePolicyFile) + } + o.ImageContentSourcePolicyList = append(o.ImageContentSourcePolicyList, *icsp) + return nil + +} + +func (o *SecurityOptions) Complete(f kcmdutil.Factory, image string) error { + o.ICSPClientFn = func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) { + // If ImageContentSourceFile is given, only add ImageContentSource from file, don't search cluster ICSP + if len(o.ImageContentSourcePolicyFile) != 0 { + return nil, nil + } + restConfig, err := f.ToRESTConfig() + if err != nil { + // may or may not be connected to a cluster + // don't error if can't connect + klog.V(4).Infof("did not connect to an OpenShift 4.x server, will not lookup ImageContentSourcePolicies: %v", err) + return nil, nil + } + icspClient, err := operatorv1alpha1client.NewForConfig(restConfig) + if err != nil { + // may or may not be connected to a cluster + // don't error if can't connect + klog.V(4).Infof("did not connect to an OpenShift 4.x server, will not lookup ImageContentSourcePolicies: %v", err) + return nil, nil + } + return icspClient.ImageContentSourcePolicies(), nil + } + return nil +} + +func (o *SecurityOptions) AddICSPsFromCluster() error { + icspClient, err := o.ICSPClientFn() + if err != nil { + return err + } + if icspClient != nil { + o.GetICSPs(icspClient) + } + return nil +} + +// GetICSPs will lookup ICSPs from cluster. Since it's not a hard requirement to find ICSPs from cluster, GetICSPs logs errors rather than returning errors. +func (o *SecurityOptions) GetICSPs(icspClient operatorv1alpha1client.ImageContentSourcePolicyInterface) { + icsps, err := icspClient.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + // may or may not have access to ICSPs in cluster + // don't error if can't access ICSPs + klog.V(4).Infof("did not access any ImageContentSourcePolicies in cluster: %v", err) + } + if len(icsps.Items) == 0 { + klog.V(4).Info("no ImageContentSourcePolicies found in cluster") + } + o.ImageContentSourcePolicyList = append(o.ImageContentSourcePolicyList, icsps.Items...) +} + +func (o *SecurityOptions) Context(image imagereference.DockerImageReference) (*registryclient.Context, error) { + if o.CachedContext != nil { + return o.CachedContext, nil + } + context, err := o.NewContext(image) + if err == nil { + o.CachedContext = context + o.CachedContext.Retries = 3 + } + return context, err +} + +func (o *SecurityOptions) NewContext(image imagereference.DockerImageReference) (*registryclient.Context, error) { + rt, err := rest.TransportFor(&rest.Config{}) + if err != nil { + return nil, err + } + insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}}) + if err != nil { + return nil, err + } + creds := dockercredentials.NewLocal() + if len(o.RegistryConfig) > 0 { + creds, err = dockercredentials.NewFromFile(o.RegistryConfig) + if err != nil { + return nil, fmt.Errorf("unable to load --registry-config: %v", err) + } + } + context := registryclient.NewContext(rt, insecureRT).WithCredentials(creds) + context.DisableDigestVerification = o.SkipVerification + if len(image.String()) > 0 { + context.AlternativeSources = &addAlternativeImageSources{} + altSources, err := context.AlternativeSources.AddImageSources(image, o.ImageContentSourcePolicyList) + if err != nil { + return nil, err + } + context.ImageSources = altSources + } + return context, nil +} + +type addAlternativeImageSources struct{} + +func (a *addAlternativeImageSources) AddImageSources(imageRef imagereference.DockerImageReference, icspList []operatorv1alpha1.ImageContentSourcePolicy) ([]imagereference.DockerImageReference, error) { + if len(icspList) == 0 { + return nil, nil + } + var imageSources []imagereference.DockerImageReference + for _, icsp := range icspList { + repoDigestMirrors := icsp.Spec.RepositoryDigestMirrors + var sourceMatches bool + for _, rdm := range repoDigestMirrors { + rdmRef, err := imagereference.Parse(rdm.Source) + if err != nil { + return nil, err + } + if imageRef.AsRepository() == rdmRef.AsRepository() { + klog.V(2).Infof("%v RepositoryDigestMirrors source matches given image", imageRef.AsRepository()) + sourceMatches = true + } + for _, m := range rdm.Mirrors { + if sourceMatches { + klog.V(2).Infof("%v RepositoryDigestMirrors mirror added to potential ImageSourcePrefixes from ImageContentSourcePolicy", m) + mRef, err := imagereference.Parse(m) + if err != nil { + return nil, err + } + imageSources = append(imageSources, mRef) + } + } + } + } + // make sure at least 1 imagesource + // ie, make sure the image passed is included in image sources + if len(imageSources) == 0 { + imageSources = append(imageSources, imageRef.AsRepository()) + } + uniqueMirrors := make([]imagereference.DockerImageReference, 0, len(imageSources)) + uniqueMap := make(map[imagereference.DockerImageReference]bool) + for _, imageSourceMirror := range imageSources { + if _, ok := uniqueMap[imageSourceMirror]; !ok { + uniqueMap[imageSourceMirror] = true + uniqueMirrors = append(uniqueMirrors, imageSourceMirror) + } + } + klog.V(2).Infof("Found sources: %v for image: %v", uniqueMirrors, imageRef) + return uniqueMirrors, nil +} diff --git a/pkg/cli/image/manifest/security_test.go b/pkg/cli/image/manifest/security_test.go new file mode 100644 index 0000000000..26f20b57af --- /dev/null +++ b/pkg/cli/image/manifest/security_test.go @@ -0,0 +1,204 @@ +package manifest + +import ( + "io/ioutil" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1" + operatorv1alpha1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1alpha1" + imagereference "github.com/openshift/library-go/pkg/image/reference" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func securityOpts(icspList []operatorv1alpha1.ImageContentSourcePolicy, icspFile string) *SecurityOptions { + return &SecurityOptions{ + Insecure: true, + SkipVerification: true, + ImageContentSourcePolicyFile: icspFile, + ImageContentSourcePolicyList: icspList, + CachedContext: nil, + } +} + +func icspFile(t *testing.T) string { + icspFile, err := ioutil.TempFile("/tmp", "test.*.icsp.yaml") + if err != nil { + t.Errorf("error creating test icsp file: %v", err) + } + icsp := ` +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: release +spec: + repositoryDigestMirrors: + - mirrors: + - someregistry/match/file + source: quay.io/ocp-test/release + - mirrors: + - someregistry/match/file + source: quay.io/ocp-test/another-source +` + err = ioutil.WriteFile(icspFile.Name(), []byte(icsp), 0) + if err != nil { + t.Errorf("error wriing to test icsp file: %v", err) + } + return icspFile.Name() +} + +func TestAlternativeImageSources(t *testing.T) { + tests := []struct { + name string + icspList []operatorv1alpha1.ImageContentSourcePolicy + icspFile string + image string + imageSourcesExpected []string + }{ + { + name: "multiple ICSPs", + icspList: []operatorv1alpha1.ImageContentSourcePolicy{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "release", + }, + Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{ + { + Source: "quay.io/multiple/icsps", + Mirrors: []string{ + "someregistry/somerepo/release", + }, + }, + { + Source: "quay.io/ocp-test/another-release", + Mirrors: []string{ + "someregistry/somerepo/release", + }, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "another", + }, + Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{ + { + Source: "quay.io/multiple/icsps", + Mirrors: []string{ + "anotherregistry/anotherrepo/release", + }, + }, + }, + }, + }, + }, + icspFile: "", + image: "quay.io/multiple/icsps:4.5", + imageSourcesExpected: []string{"someregistry/somerepo/release", "anotherregistry/anotherrepo/release"}, + }, + { + name: "sources match ICSP file", + icspList: []operatorv1alpha1.ImageContentSourcePolicy{}, + icspFile: icspFile(t), + image: "quay.io/ocp-test/release:4.6", + imageSourcesExpected: []string{"someregistry/match/file"}, + }, + { + name: "no match ICSP file", + icspList: []operatorv1alpha1.ImageContentSourcePolicy{}, + icspFile: icspFile(t), + image: "quay.io/passed/image:4.5", + imageSourcesExpected: []string{"quay.io/passed/image"}, + }, + { + name: "ICSP mirrors match image", + icspList: []operatorv1alpha1.ImageContentSourcePolicy{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "release", + }, + Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{ + { + Source: "quay.io/ocp-test/release", + Mirrors: []string{ + "someregistry/mirrors/match", + }, + }, + }, + }, + }, + }, + icspFile: "", + image: "quay.io/ocp-test/release:4.5", + imageSourcesExpected: []string{"someregistry/mirrors/match"}, + }, + { + name: "ICSP source matches image", + icspList: []operatorv1alpha1.ImageContentSourcePolicy{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "release", + }, + Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{ + { + Source: "quay.io/source/matches", + Mirrors: []string{ + "someregistry/somerepo/release", + }, + }, + }, + }, + }, + }, + icspFile: "", + image: "quay.io/source/matches:4.5", + imageSourcesExpected: []string{"someregistry/somerepo/release"}, + }, + { + name: "no ICSP", + icspList: nil, + icspFile: "", + image: "quay.io/ocp-test/release:4.5", + imageSourcesExpected: []string{}, + }, + } + for _, tt := range tests { + imageRef, err := imagereference.Parse(tt.image) + if err != nil { + t.Errorf("parsing image reference error = %v", err) + } + secOpts := securityOpts(tt.icspList, tt.icspFile) + secOpts.ICSPClientFn = func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) { + return nil, nil + } + var expectedRefs []imagereference.DockerImageReference + for _, expected := range tt.imageSourcesExpected { + expectedRef, err := imagereference.Parse(expected) + if err != nil { + t.Errorf("parsing image reference error = %v", err) + } + expectedRefs = append(expectedRefs, expectedRef) + } + + if len(tt.icspFile) > 0 { + err := secOpts.AddImageSourcePoliciesFromFile(tt.image) + if err != nil { + t.Errorf("add ICSP from file error = %v", err) + } + } + a := &addAlternativeImageSources{} + altSources, err := a.AddImageSources(imageRef, secOpts.ImageContentSourcePolicyList) + if err != nil { + t.Errorf("registry client Context error = %v", err) + } + if !reflect.DeepEqual(expectedRefs, altSources) { + t.Errorf("AddAlternativeImageSource got = %v, want %v, diff = %v", altSources, expectedRefs, cmp.Diff(altSources, expectedRefs)) + } + } +} diff --git a/pkg/cli/image/mirror/mirror.go b/pkg/cli/image/mirror/mirror.go index bc57d44497..3ee06a676d 100644 --- a/pkg/cli/image/mirror/mirror.go +++ b/pkg/cli/image/mirror/mirror.go @@ -26,6 +26,7 @@ import ( kcmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" + imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" "github.com/openshift/oc/pkg/cli/image/imagesource" imagemanifest "github.com/openshift/oc/pkg/cli/image/manifest" @@ -187,7 +188,7 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { o.KeepManifestList = true } - registryContext, err := o.SecurityOptions.Context() + registryContext, err := o.SecurityOptions.Context(imagereference.DockerImageReference{}) if err != nil { return err } @@ -209,6 +210,7 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { if err != nil { return err } + for _, filename := range o.Filenames { mappings, err := parseFile(filename, overlap, o.In, opts.ExpandWildcard) if err != nil { @@ -230,17 +232,18 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { return nil } -func (o *MirrorImageOptions) Repository(ctx context.Context, context *registryclient.Context, ref imagesource.TypedImageReference, source bool) (distribution.Repository, error) { +func (o *MirrorImageOptions) Repository(ctx context.Context, regContext *registryclient.Context, ref imagesource.TypedImageReference, source bool) (distribution.Repository, distribution.ManifestService, error) { dir := o.FileDir if len(o.FromFileDir) > 0 && source { dir = o.FromFileDir } klog.V(5).Infof("Find source=%t registry with %#v", source, ref) + opts := &imagesource.Options{ FileDir: dir, Insecure: o.SecurityOptions.Insecure, AttemptS3BucketCopy: o.AttemptS3BucketCopy, - RegistryContext: context, + RegistryContext: regContext, } return opts.Repository(ctx, ref) } @@ -430,7 +433,7 @@ type contextKey struct { func (o *MirrorImageOptions) plan() (*plan, error) { ctx := apirequest.NewContext() - context, err := o.SecurityOptions.Context() + context, err := o.SecurityOptions.Context(imagereference.DockerImageReference{}) if err != nil { return nil, err } @@ -463,16 +466,12 @@ func (o *MirrorImageOptions) plan() (*plan, error) { for name := range tree { src := tree[name] q.Queue(func(_ workqueue.Work) { - srcRepo, err := o.Repository(ctx, fromContext, src.ref, true) + srcRepo, manifests, err := o.Repository(ctx, fromContext, src.ref, true) if err != nil { plan.AddError(retrieverError{err: fmt.Errorf("unable to connect to %s: %v", src.ref, err), src: src.ref}) return } - manifests, err := srcRepo.Manifests(ctx) - if err != nil { - plan.AddError(retrieverError{src: src.ref, err: fmt.Errorf("unable to access source image %s manifests: %v", src.ref, err)}) - return - } + rq := registryWorkers[name.registry] rq.Batch(func(w workqueue.Work) { // convert source tags to digests @@ -533,11 +532,12 @@ func (o *MirrorImageOptions) plan() (*plan, error) { for _, dst := range pushTargets { var toRepo distribution.Repository + var toManifests distribution.ManifestService var err error if o.DryRun { toRepo, err = imagesource.NewDryRun(dst.ref) } else { - toRepo, err = o.Repository(ctx, toContexts[contextKeyForReference(dst.ref)], dst.ref, false) + toRepo, toManifests, err = o.Repository(ctx, toContexts[contextKeyForReference(dst.ref)], dst.ref, false) } if err != nil { plan.AddError(retrieverError{src: src.ref, dst: dst.ref, err: fmt.Errorf("unable to connect to %s: %v", dst.ref, err)}) @@ -550,12 +550,6 @@ func (o *MirrorImageOptions) plan() (*plan, error) { repoPlan := registryPlan.RepositoryPlan(canonicalTo.String()) blobPlan := repoPlan.Blobs(src.ref, location) - toManifests, err := toRepo.Manifests(ctx) - if err != nil { - repoPlan.AddError(retrieverError{src: src.ref, dst: dst.ref, err: fmt.Errorf("unable to access destination image %s manifests: %v", src.ref, err)}) - continue - } - var mustCopyLayers bool switch { case o.Force: From 1e7d78dd9e99a5c0a9b66bba9043728010d87739 Mon Sep 17 00:00:00 2001 From: Sally O'Malley Date: Mon, 16 Nov 2020 21:55:20 -0500 Subject: [PATCH 2/2] add --lookup-cluster-icsp flag and alternative image awareness to 'oc image mirror|extract|info' --- contrib/completions/bash/oc | 38 +++ contrib/completions/zsh/oc | 38 +++ pkg/cli/admin/release/extract.go | 39 +-- pkg/cli/admin/release/extract_tools.go | 22 +- pkg/cli/admin/release/info.go | 121 ++------ pkg/cli/admin/release/mirror.go | 42 +-- pkg/cli/admin/release/new.go | 6 + .../admin/verifyimagesignature/manifest.go | 4 +- pkg/cli/image/append/append.go | 35 +-- pkg/cli/image/extract/extract.go | 25 +- pkg/cli/image/image.go | 6 +- pkg/cli/image/imagesource/file.go | 13 +- pkg/cli/image/imagesource/options.go | 51 +--- pkg/cli/image/imagesource/s3.go | 16 +- pkg/cli/image/info/info.go | 22 +- pkg/cli/image/manifest/.security_test.go.swp | Bin 16384 -> 0 bytes pkg/cli/image/manifest/manifest.go | 1 + pkg/cli/image/manifest/security.go | 268 ++++++++++++------ pkg/cli/image/manifest/security_test.go | 23 +- pkg/cli/image/mirror/mirror.go | 49 ++-- 20 files changed, 452 insertions(+), 367 deletions(-) delete mode 100644 pkg/cli/image/manifest/.security_test.go.swp diff --git a/contrib/completions/bash/oc b/contrib/completions/bash/oc index 4243d368ac..4b8c2fac84 100644 --- a/contrib/completions/bash/oc +++ b/contrib/completions/bash/oc @@ -576,8 +576,13 @@ _oc_adm_catalog_build() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--manifest-dir=") two_word_flags+=("--manifest-dir") local_nonpersistent_flags+=("--manifest-dir=") @@ -665,11 +670,16 @@ _oc_adm_catalog_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--icsp-scope=") two_word_flags+=("--icsp-scope") local_nonpersistent_flags+=("--icsp-scope=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--manifests-only") local_nonpersistent_flags+=("--manifests-only") flags+=("--max-components=") @@ -5449,6 +5459,8 @@ _oc_adm_release_extract() local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5558,6 +5570,8 @@ _oc_adm_release_info() local_nonpersistent_flags+=("--include-images") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5657,6 +5671,8 @@ _oc_adm_release_mirror() local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5777,11 +5793,16 @@ _oc_adm_release_new() flags+=("--from-release=") two_word_flags+=("--from-release") local_nonpersistent_flags+=("--from-release=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--include=") two_word_flags+=("--include") local_nonpersistent_flags+=("--include=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--mapping-file=") two_word_flags+=("--mapping-file") local_nonpersistent_flags+=("--mapping-file=") @@ -13420,11 +13441,16 @@ _oc_image_append() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image=") two_word_flags+=("--image") local_nonpersistent_flags+=("--image=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -13518,6 +13544,8 @@ _oc_image_extract() local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--only-files") local_nonpersistent_flags+=("--only-files") flags+=("--path=") @@ -13596,8 +13624,13 @@ _oc_image_info() flags+=("--filter-by-os=") two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--output=") two_word_flags+=("--output") two_word_flags+=("-o") @@ -13685,10 +13718,15 @@ _oc_image_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--keep-manifest-list") local_nonpersistent_flags+=("--keep-manifest-list") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") diff --git a/contrib/completions/zsh/oc b/contrib/completions/zsh/oc index 5ebd35740f..032aa204a6 100644 --- a/contrib/completions/zsh/oc +++ b/contrib/completions/zsh/oc @@ -676,8 +676,13 @@ _oc_adm_catalog_build() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--manifest-dir=") two_word_flags+=("--manifest-dir") local_nonpersistent_flags+=("--manifest-dir=") @@ -765,11 +770,16 @@ _oc_adm_catalog_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--icsp-scope=") two_word_flags+=("--icsp-scope") local_nonpersistent_flags+=("--icsp-scope=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--manifests-only") local_nonpersistent_flags+=("--manifests-only") flags+=("--max-components=") @@ -5549,6 +5559,8 @@ _oc_adm_release_extract() local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5658,6 +5670,8 @@ _oc_adm_release_info() local_nonpersistent_flags+=("--include-images") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5757,6 +5771,8 @@ _oc_adm_release_mirror() local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5877,11 +5893,16 @@ _oc_adm_release_new() flags+=("--from-release=") two_word_flags+=("--from-release") local_nonpersistent_flags+=("--from-release=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--include=") two_word_flags+=("--include") local_nonpersistent_flags+=("--include=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--mapping-file=") two_word_flags+=("--mapping-file") local_nonpersistent_flags+=("--mapping-file=") @@ -13520,11 +13541,16 @@ _oc_image_append() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image=") two_word_flags+=("--image") local_nonpersistent_flags+=("--image=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -13618,6 +13644,8 @@ _oc_image_extract() local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--only-files") local_nonpersistent_flags+=("--only-files") flags+=("--path=") @@ -13696,8 +13724,13 @@ _oc_image_info() flags+=("--filter-by-os=") two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--output=") two_word_flags+=("--output") two_word_flags+=("-o") @@ -13785,10 +13818,15 @@ _oc_image_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--keep-manifest-list") local_nonpersistent_flags+=("--keep-manifest-list") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") diff --git a/pkg/cli/admin/release/extract.go b/pkg/cli/admin/release/extract.go index 6a791e16be..89c86ff92d 100644 --- a/pkg/cli/admin/release/extract.go +++ b/pkg/cli/admin/release/extract.go @@ -102,7 +102,6 @@ func NewExtract(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra. } flags := cmd.Flags() o.SecurityOptions.Bind(flags) - o.FilterOptions.Bind(flags) o.ParallelOptions.Bind(flags) flags.StringVar(&o.From, "from", o.From, "Image containing the release payload.") @@ -128,9 +127,7 @@ type ExtractOptions struct { genericclioptions.IOStreams SecurityOptions imagemanifest.SecurityOptions - FilterOptions imagemanifest.FilterOptions ParallelOptions imagemanifest.ParallelOptions - InfoOptions InfoOptions Output string @@ -177,14 +174,11 @@ func (o *ExtractOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args [ } o.From = args[0] - if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { + if err := o.SecurityOptions.Complete(f); err != nil { return err } - if err := o.InfoOptions.Complete(f, cmd, args); err != nil { - return err - } - if err := o.SecurityOptions.Complete(f, o.From); err != nil { - return err + if !o.SecurityOptions.LookupClusterICSP && len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.TryAlternativeSources = true } return nil } @@ -220,13 +214,6 @@ func (o *ExtractOptions) Run() error { } } - opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) - opts.ParallelOptions = o.ParallelOptions - opts.FilterOptions = o.FilterOptions - opts.SecurityOptions = o.SecurityOptions - - opts.FileDir = o.FileDir - switch { case sources > 1: return fmt.Errorf("only one of --tools, --command, --credentials-requests, --file, or --git may be specified") @@ -253,8 +240,22 @@ func (o *ExtractOptions) Run() error { if err != nil { return err } + opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) + opts.ParallelOptions = o.ParallelOptions + opts.SecurityOptions = o.SecurityOptions + opts.FileDir = o.FileDir + switch { + // o.File > 0 when mirroring release case len(o.File) > 0: + // set ref if using alternative image sources + regContext, err := opts.SecurityOptions.Context() + if err != nil { + return err + } + newRef, _, _, err := opts.SecurityOptions.PreferredImageSource(ref.Ref, regContext) + ref.Ref = newRef + src = ref.String() if o.ImageMetadataCallback != nil { opts.ImageMetadataCallback = o.ImageMetadataCallback } @@ -433,7 +434,11 @@ func (o *ExtractOptions) extractGit(dir string) error { return err } - release, err := o.InfoOptions.LoadReleaseInfo(o.From, false, false, "") + opts := NewInfoOptions(o.IOStreams) + opts.SecurityOptions = o.SecurityOptions + opts.FileDir = o.FileDir + + release, err := opts.LoadReleaseInfo(o.From, false, false, "") if err != nil { return err } diff --git a/pkg/cli/admin/release/extract_tools.go b/pkg/cli/admin/release/extract_tools.go index 766a9d8eaa..2a453eebdf 100644 --- a/pkg/cli/admin/release/extract_tools.go +++ b/pkg/cli/admin/release/extract_tools.go @@ -292,24 +292,28 @@ func (o *ExtractOptions) extractCommand(command string) error { } } - // load release image + // load the release image dir := o.Directory + infoOptions := NewInfoOptions(o.IOStreams) + infoOptions.SecurityOptions = o.SecurityOptions + infoOptions.FileDir = o.FileDir + var releaseFromRefErr error targetRelease := targetReleaseInfo{} - if len(o.SecurityOptions.ImageContentSourcePolicyFile) > 0 { - o.SecurityOptions.AddImageSourcePoliciesFromFile(o.From) - releaseFromImageSources, err := o.InfoOptions.LoadReleaseInfo(o.From, false, true, o.SecurityOptions.ImageContentSourcePolicyFile) + if len(o.SecurityOptions.ICSPFile) > 0 { + o.SecurityOptions.AddImageSourcePoliciesFromFile() + releaseFromImageSources, err := infoOptions.LoadReleaseInfo(o.From, false, true, o.SecurityOptions.ICSPFile) if err != nil { - return fmt.Errorf("could not load release %s from icsp file %s: %v", o.From, o.SecurityOptions.ImageContentSourcePolicyFile, err) + return fmt.Errorf("could not load release %s from icsp file %s: %v", o.From, o.SecurityOptions.ICSPFile, err) } targetRelease, err = o.setReleaseLookup(releaseFromImageSources, targets, currentOS, willArchive) if err != nil { // if icsp-file set, then return error if lookup from icsp fails - return fmt.Errorf("failed lookup of release %s from icsp file %s: %v", o.From, o.SecurityOptions.ImageContentSourcePolicyFile, err) + return fmt.Errorf("failed lookup of release %s from icsp file %s: %v", o.From, o.SecurityOptions.ICSPFile, err) } } else { // This will be tried first, if no ImageContentSourcePolicyFile passed - releaseFromRef, loadReleaseFromRefErr := o.InfoOptions.LoadReleaseInfo(o.From, false, false, "") + releaseFromRef, loadReleaseFromRefErr := infoOptions.LoadReleaseInfo(o.From, false, false, "") if loadReleaseFromRefErr == nil { targetRelease, releaseFromRefErr = o.setReleaseLookup(releaseFromRef, targets, currentOS, willArchive) // This will be returned if further lookup fails @@ -317,7 +321,7 @@ func (o *ExtractOptions) extractCommand(command string) error { // If there's an error, now look for other imageSources - icsp and/or the user-passed image registry/repo/name klog.V(2).Infof("Failed lookup of release from its reference: %v", releaseFromRefErr) // now try other sources - releaseFromImageSources, err := o.InfoOptions.LoadReleaseInfo(o.From, false, true, "") + releaseFromImageSources, err := infoOptions.LoadReleaseInfo(o.From, false, true, "") if err != nil { return err } @@ -330,7 +334,7 @@ func (o *ExtractOptions) extractCommand(command string) error { } else { klog.V(2).Infof("Failed to load release info from image reference: %v", loadReleaseFromRefErr) // now try other sources - releaseFromImageSources, err := o.InfoOptions.LoadReleaseInfo(o.From, false, true, "") + releaseFromImageSources, err := infoOptions.LoadReleaseInfo(o.From, false, true, "") if err != nil { return err } diff --git a/pkg/cli/admin/release/info.go b/pkg/cli/admin/release/info.go index fce39bbca6..f8ef31cc1e 100644 --- a/pkg/cli/admin/release/info.go +++ b/pkg/cli/admin/release/info.go @@ -367,9 +367,12 @@ func (o *InfoOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []st o.From = o.Images[0] o.Images = o.Images[1:] } - if err = o.SecurityOptions.Complete(f, ""); err != nil { + if err := o.SecurityOptions.Complete(f); err != nil { return err } + if !o.SecurityOptions.LookupClusterICSP && len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.TryAlternativeSources = true + } return nil } @@ -738,6 +741,7 @@ func (i *ReleaseInfo) Platform() string { } func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool, setImageSourcePrefix bool, icspFile string) (*ReleaseInfo, error) { + replacedRef := false opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) opts.SecurityOptions = o.SecurityOptions opts.FileDir = o.FileDir @@ -746,68 +750,19 @@ func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool, setImag return nil, err } - fromContext, err := opts.SecurityOptions.Context(ref.Ref) + fromContext, err := opts.SecurityOptions.Context() if err != nil { return nil, err } - sourceOpts := &imagesource.Options{ - FileDir: o.FileDir, - Insecure: o.SecurityOptions.Insecure, - RegistryContext: fromContext, - } - if setImageSourcePrefix { - if len(icspFile) > 0 { - opts.SecurityOptions.ImageContentSourcePolicyFile = icspFile - err := opts.SecurityOptions.AddImageSourcePoliciesFromFile(image) - if err != nil { - return nil, err - } - altSources, err := fromContext.AddImageSources(ref.Ref, opts.SecurityOptions.ImageContentSourcePolicyList) - if err != nil { - return nil, err - } - sourceOpts.RegistryContext.ImageSources = altSources - } else { - // now try to look for ICSPs from cluster - // only look for ICSP if release lookup from image reference fails - // such as when working with mirrored registry - if err := opts.SecurityOptions.AddICSPsFromCluster(); err != nil { - return nil, err - } - altSources, err := fromContext.AddImageSources(ref.Ref, opts.SecurityOptions.ImageContentSourcePolicyList) - if err != nil { - return nil, err - } - sourceOpts.RegistryContext.ImageSources = altSources - } - // Try alternative image sources rather than only the single reference source - // imageSources is a slice of 'registry/repo/name' from ImageContentSourcePolicies - var err error - imageRef, err := imagereference.Parse(image) - if err != nil { - return nil, err - } - for _, icsRef := range sourceOpts.RegistryContext.ImageSources { - icsRef.ID = imageRef.ID - icsRef.Tag = imageRef.Tag - ref, err = imagesource.ParseReference(icsRef.String()) - if err != nil { - return nil, err - } - if _, _, err = sourceOpts.Repository(context.TODO(), ref); err == nil { - image = icsRef.String() - break - } - } + if setImageSourcePrefix { + newRef, _, _, err := opts.SecurityOptions.PreferredImageSource(ref.Ref, fromContext) if err != nil { return nil, err } - } - - ref, err = imagesource.ParseReference(image) - if err != nil { - return nil, err + replacedRef = true + image = newRef.String() + ref.Ref = newRef } verifier := imagemanifest.NewVerifier() @@ -850,54 +805,22 @@ func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool, setImag errs = append(errs, err) return true, nil } - if setImageSourcePrefix { - userGivenRef := ref.Ref.AsRepository() - imageSet := false + if replacedRef { + sourceOpts := &imagesource.Options{FileDir: opts.FileDir, Insecure: opts.SecurityOptions.Insecure, RegistryContext: fromContext} for _, tag := range is.Spec.Tags { - // If useImageContentSources true, try every one of imageSources rather than it's single reference. - // imagereference.Parse returns the digest ID of each component in the release image-reference. // If can't get digest ID, skip this tag, this happens when user has built a payload by // replacing component images in the release with a new image // imageSources is slice of 'registry/repo/name' determined from ICSP - for _, icsRef := range fromContext.ImageSources { - tagRef, err := imagereference.Parse(tag.From.Name) - // if err != nil, skip this tag - if err == nil { - icsRef.ID = tagRef.ID - srcRef, err := imagesource.ParseReference(icsRef.String()) - if err != nil { - return true, err - } - if _, _, err = sourceOpts.Repository(context.TODO(), srcRef); err == nil { - tag.From.Name = icsRef.String() - imageSet = true - // ignore error, if there's an error don't substitute - break - } + tagRef, err := imagereference.Parse(tag.From.Name) + // if err != nil, skip this tag + if err == nil { + ref.Ref.ID = tagRef.ID + srcRef, err := imagesource.ParseReference(ref.String()) + if err != nil { + return true, err } - } - if !imageSet { - // if user passed the flag to use an iCSP file but no image was set from this ICSP file, error - if len(o.SecurityOptions.ImageContentSourcePolicyFile) > 0 { - return false, fmt.Errorf("could not find image source from ImageContentSourceFile %v", o.SecurityOptions.ImageContentSourcePolicyFile) - } - // Now try the registry/repo passed from the user. If the image does not exist, - // proceed with the release info from the release image-references, from readReleaseImageReference above - tagRef, err := imagereference.Parse(tag.From.Name) - // if err != nil, skip this tag - if err == nil { - userGivenRef.ID = tagRef.ID - userGivenTypedRef, err := imagesource.ParseReference(userGivenRef.String()) - if err != nil { - return true, err - } - // If the user-given registry/repo/name:digest exists, replace with that, if not keep the - // is.Spec.Tag from release image-reference - // if userGivenImage:digestID exists, set that. If not, return original error - if _, _, err = sourceOpts.Repository(context.TODO(), userGivenTypedRef); err == nil { - // ignore error, if there's an error don't substitute - tag.From.Name = userGivenRef.String() - } + if _, _, err = sourceOpts.RepositoryWithManifests(context.TODO(), srcRef); err == nil { + tag.From.Name = ref.String() } } } diff --git a/pkg/cli/admin/release/mirror.go b/pkg/cli/admin/release/mirror.go index 4a4dd00689..21e4436e0c 100644 --- a/pkg/cli/admin/release/mirror.go +++ b/pkg/cli/admin/release/mirror.go @@ -195,8 +195,7 @@ type MirrorOptions struct { ReleaseImageICSPToDir string Overwrite bool - DryRun bool - PrintImageContentInstructions bool + DryRun bool ImageClientFn func() (imageclient.Interface, string, error) CoreV1ClientFn func() (corev1client.ConfigMapInterface, error) @@ -226,9 +225,12 @@ func (o *MirrorOptions) Complete(cmd *cobra.Command, f kcmdutil.Factory, args [] } o.From = args[0] - if err := o.SecurityOptions.Complete(f, ""); err != nil { + if err := o.SecurityOptions.Complete(f); err != nil { return err } + if !o.SecurityOptions.LookupClusterICSP && len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.TryAlternativeSources = true + } o.ImageClientFn = func() (imageclient.Interface, string, error) { cfg, err := f.ToRESTConfig() @@ -563,7 +565,7 @@ func (o *MirrorOptions) Run() error { // if the source ref is a file type, provide a function that checks the local file store for a given manifest // before continuing, to allow mirroring an entire release to disk in a single file://REPO. if srcRef.Type == imagesource.DestinationFile { - if _, manifests, err := (&imagesource.Options{FileDir: o.FromDir}).Repository(context.TODO(), srcRef); err == nil { + if _, manifests, err := (&imagesource.Options{FileDir: o.FromDir}).RepositoryWithManifests(context.TODO(), srcRef); err == nil { sourceFn = func(ref imagesource.TypedImageReference) imagesource.TypedImageReference { if ref.Type == imagesource.DestinationFile || len(ref.Ref.ID) == 0 { return ref @@ -610,7 +612,6 @@ func (o *MirrorOptions) Run() error { repositories := make(map[string]struct{}) // build the mapping list for mirroring and rewrite if necessary - //userGivenRef := srcRef.Ref.AsRepository() for i := range is.Spec.Tags { tag := &is.Spec.Tags[i] if tag.From == nil || tag.From.Kind != "DockerImage" { @@ -624,10 +625,6 @@ func (o *MirrorOptions) Run() error { return fmt.Errorf("image-references should only contain pointers to images by digest: %s", tag.From.Name) } - opts := mirror.NewMirrorImageOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) - opts.SecurityOptions = o.SecurityOptions - opts.ParallelOptions = o.ParallelOptions - opts.FileDir = o.ToDir // Allow mirror refs to be sourced locally srcMirrorRef := imagesource.TypedImageReference{Ref: from, Type: imagesource.DestinationRegistry} srcMirrorRef = sourceFn(srcMirrorRef) @@ -821,6 +818,7 @@ func (o *MirrorOptions) Run() error { if len(o.ReleaseImageICSPToDir) == 0 { o.ReleaseImageICSPToDir = configFilesBaseDir } + if err := o.printImageContentInstructions(repositories, toList, releaseDigest); err != nil { return fmt.Errorf("Error creating mirror usage instructions: %v", err) } @@ -863,27 +861,29 @@ func (o *MirrorOptions) printImageContentInstructions(repositories map[string]st var sources []operatorv1alpha1.RepositoryDigestMirrors + var mirrorRef imagesource.TypedImageReference + var err error for _, to := range toList { - mirrorRef, err := imagesource.ParseReference(to) + mirrorRef, err = imagesource.ParseReference(to) if err != nil { return fmt.Errorf("Unable to parse image reference '%s': %v", to, err) - if mirrorRef.Type != imagesource.DestinationRegistry { - return nil - } - mirrorRepo := mirrorRef.Ref.AsRepository().String() - - if len(o.From) != 0 { - sourceRef, err := imagesource.ParseReference(o.From) - if err != nil { - return fmt.Errorf("Unable to parse image reference '%s': %v", o.From, err) } + if mirrorRef.Type != imagesource.DestinationRegistry { + return nil + } + mirrorRepo := mirrorRef.Ref.AsRepository().String() + + if len(o.From) != 0 { + sourceRef, err := imagesource.ParseReference(o.From) + if err != nil { + return fmt.Errorf("Unable to parse image reference '%s': %v", o.From, err) + } if sourceRef.Type != imagesource.DestinationRegistry { return nil } sourceRepo := sourceRef.Ref.AsRepository().String() repositories[sourceRepo] = struct{}{} } - if len(repositories) == 0 { return nil } @@ -898,7 +898,6 @@ func (o *MirrorOptions) printImageContentInstructions(repositories map[string]st return sources[i].Source < sources[j].Source }) } -} // Create and display install-config.yaml example imageContentSources := installConfigSubsection{ @@ -911,6 +910,7 @@ func (o *MirrorOptions) printImageContentInstructions(repositories map[string]st fmt.Fprintf(o.Out, string(installConfigExample)) // Create and display ImageContentSourcePolicy + // last mirrorRef will be used to name icsp mirrorRepoStripped := strings.FieldsFunc(mirrorRef.Ref.Name, func(r rune) bool { return strings.ContainsRune(" .:/", r) }) icspName := strings.Join(mirrorRepoStripped[:], "-") diff --git a/pkg/cli/admin/release/new.go b/pkg/cli/admin/release/new.go index 7bd37cd880..0f1455763e 100644 --- a/pkg/cli/admin/release/new.go +++ b/pkg/cli/admin/release/new.go @@ -246,6 +246,12 @@ func (o *NewOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []str o.Namespace = namespace } } + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } + if !o.SecurityOptions.LookupClusterICSP && len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.TryAlternativeSources = true + } return nil } diff --git a/pkg/cli/admin/verifyimagesignature/manifest.go b/pkg/cli/admin/verifyimagesignature/manifest.go index e0b0c16504..a3d5816566 100644 --- a/pkg/cli/admin/verifyimagesignature/manifest.go +++ b/pkg/cli/admin/verifyimagesignature/manifest.go @@ -31,12 +31,10 @@ func getImageManifestByIDFromRegistry(registry *url.URL, repositoryName, imageID if err != nil { return nil, err } - - manifests, err := repo.Manifests(ctx, nil) + manifests, err := repo.Manifests(ctx) if err != nil { return nil, err } - manifest, err := manifests.Get(ctx, godigest.Digest(imageID)) if err != nil { return nil, err diff --git a/pkg/cli/image/append/append.go b/pkg/cli/image/append/append.go index 087ff11426..5ec6c52717 100644 --- a/pkg/cli/image/append/append.go +++ b/pkg/cli/image/append/append.go @@ -107,7 +107,7 @@ func NewAppendImageOptions(streams genericclioptions.IOStreams) *AppendImageOpti } // New creates a new command -func NewCmdAppendImage(streams genericclioptions.IOStreams) *cobra.Command { +func NewCmdAppendImage(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewAppendImageOptions(streams) cmd := &cobra.Command{ @@ -116,7 +116,7 @@ func NewCmdAppendImage(streams genericclioptions.IOStreams) *cobra.Command { Long: desc, Example: example, Run: func(c *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(c, args)) + kcmdutil.CheckErr(o.Complete(f, c, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -145,11 +145,13 @@ func NewCmdAppendImage(streams genericclioptions.IOStreams) *cobra.Command { return cmd } -func (o *AppendImageOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *AppendImageOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { return err } - + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } for _, arg := range args { if arg == "-" { if o.LayerStream != nil { @@ -210,30 +212,22 @@ func (o *AppendImageOptions) Run() error { } ctx := context.Background() - fromContext, err := o.SecurityOptions.Context(from.Ref) + if len(o.FromFileDir) > 0 { + o.SecurityOptions.FileDir = o.FromFileDir + } + + fromContext, err := o.SecurityOptions.Context() if err != nil { return err } - fromOptions := &imagesource.Options{ - FileDir: o.FileDir, - Insecure: o.SecurityOptions.Insecure, - RegistryContext: fromContext, - } - if len(o.FromFileDir) > 0 { - fromOptions.FileDir = o.FromFileDir - } toContext := fromContext.Copy().WithActions("pull", "push") - toOptions := &imagesource.Options{ - FileDir: o.FileDir, - Insecure: o.SecurityOptions.Insecure, - RegistryContext: toContext, - } - toRepo, toManifests, err := toOptions.Repository(ctx, to) + newImageRef, toRepo, toManifests, err := o.SecurityOptions.PreferredImageSource(to.Ref, fromContext) if err != nil { return err } + to.Ref = newImageRef var ( base *dockerv1client.DockerImageConfig baseDigest digest.Digest @@ -244,10 +238,11 @@ func (o *AppendImageOptions) Run() error { manifestLocation imagemanifest.ManifestLocation ) if from != nil { - repo, _, err := toOptions.Repository(ctx, *from) + newFromRef, repo, _, err := o.SecurityOptions.PreferredImageSource(from.Ref, toContext) if err != nil { return err } + from.Ref = newFromRef srcManifest, manifestLocation, err = imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) if err != nil { return fmt.Errorf("unable to read image %s: %v", from, err) diff --git a/pkg/cli/image/extract/extract.go b/pkg/cli/image/extract/extract.go index 4478737567..8632fcf464 100644 --- a/pkg/cli/image/extract/extract.go +++ b/pkg/cli/image/extract/extract.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/openshift/library-go/pkg/image/dockerv1client" + imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" "github.com/openshift/oc/pkg/cli/image/archive" "github.com/openshift/oc/pkg/cli/image/imagesource" @@ -304,7 +305,7 @@ func (o *ExtractOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args [ return err } - if err := o.SecurityOptions.Complete(f, ""); err != nil { + if err := o.SecurityOptions.Complete(f); err != nil { return err } @@ -321,28 +322,25 @@ func (o *ExtractOptions) Validate() error { func (o *ExtractOptions) Run() error { ctx := context.Background() var err error - fromContext, err := o.SecurityOptions.Context(o.Mappings[0].ImageRef.Ref) + fromContext, err := o.SecurityOptions.Context() if err != nil { return err } - fromOptions := &imagesource.Options{ - FileDir: o.FileDir, - Insecure: o.SecurityOptions.Insecure, - RegistryContext: fromContext, - } stopCh := make(chan struct{}) defer close(stopCh) q := workqueue.New(o.ParallelOptions.MaxPerRegistry, stopCh) return q.Try(func(q workqueue.Try) { + var repo distribution.Repository for i := range o.Mappings { mapping := o.Mappings[i] from := mapping.ImageRef q.Try(func() error { - repo, _, err := fromOptions.Repository(ctx, from) - if err != nil || repo == nil { - return fmt.Errorf("unable to connect to image repository %s: %v", from, err) + var newImageRef imagereference.DockerImageReference + newImageRef, repo, _, err = o.SecurityOptions.PreferredImageSource(from.Ref, fromContext) + if err != nil { + return err } - + from.Ref = newImageRef srcManifest, location, err := imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) if err != nil { if imagemanifest.IsImageForbidden(err) { @@ -355,14 +353,17 @@ func (o *ExtractOptions) Run() error { } return fmt.Errorf("unable to read image %s: %v", from, err) } + contentDigest, err := registryclient.ContentDigestForManifest(srcManifest, location.Manifest.Algorithm()) if err != nil { return err } + imageConfig, layers, err := imagemanifest.ManifestToImageConfig(ctx, srcManifest, repo.Blobs(ctx), location) if err != nil { - return err + return fmt.Errorf("unable to parse image %s: %v", from, err) } + if mapping.ConditionFn != nil { ok, err := mapping.ConditionFn(&mapping, location.Manifest, imageConfig) if err != nil { diff --git a/pkg/cli/image/image.go b/pkg/cli/image/image.go index ded56d30da..f6d860c176 100644 --- a/pkg/cli/image/image.go +++ b/pkg/cli/image/image.go @@ -35,15 +35,15 @@ func NewCmdImage(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra { Message: "View or copy images:", Commands: []*cobra.Command{ - info.NewInfo(streams), - mirror.NewCmdMirrorImage(streams), + info.NewInfo(f, streams), + mirror.NewCmdMirrorImage(f, streams), }, }, { Message: "Advanced commands:", Commands: []*cobra.Command{ serve.NewServe(streams), - append.NewCmdAppendImage(streams), + append.NewCmdAppendImage(f, streams), extract.NewExtract(f, streams), }, }, diff --git a/pkg/cli/image/imagesource/file.go b/pkg/cli/image/imagesource/file.go index 9d2fb77b50..c37df7b3e3 100644 --- a/pkg/cli/image/imagesource/file.go +++ b/pkg/cli/image/imagesource/file.go @@ -26,16 +26,16 @@ type fileDriver struct { BaseDir string } -func (d *fileDriver) Repository(ctx context.Context, server *url.URL, repoName string, insecure bool) (distribution.Repository, error) { +func (d *fileDriver) Repository(ctx context.Context, server *url.URL, repoName string, insecure bool) (distribution.Repository, distribution.ManifestService, error) { klog.V(3).Infof("Repository %s %s", server, repoName) ref, err := reference.Parse(repoName) if err != nil { - return nil, err + return nil, nil, err } named, ok := ref.(reference.Named) if !ok { - return nil, fmt.Errorf("%s is not a valid repository name", repoName) + return nil, nil, fmt.Errorf("%s is not a valid repository name", repoName) } repo := &fileRepository{ @@ -43,7 +43,12 @@ func (d *fileDriver) Repository(ctx context.Context, server *url.URL, repoName s repoPath: repoPathForName(repoName), basePath: d.BaseDir, } - return repo, nil + + manifests, err := repo.Manifests(ctx) + if err != nil { + return nil, nil, fmt.Errorf("unable to get local manifest service: %v", err) + } + return repo, manifests, nil } func repoPathForName(repoName string) string { diff --git a/pkg/cli/image/imagesource/options.go b/pkg/cli/image/imagesource/options.go index e01e5bd635..caa09d7ed9 100644 --- a/pkg/cli/image/imagesource/options.go +++ b/pkg/cli/image/imagesource/options.go @@ -3,7 +3,6 @@ package imagesource import ( "context" "fmt" - "net/url" "github.com/docker/distribution" "github.com/openshift/library-go/pkg/image/registryclient" @@ -19,74 +18,36 @@ type Options struct { } // Repository retrieves the appropriate repository implementation and ManifestService for the given typed reference. -func (o *Options) Repository(ctx context.Context, ref TypedImageReference) (distribution.Repository, distribution.ManifestService, error) { - o.RegistryContext.RepositoryRetriever = ®istryContext{o.RegistryContext} +func (o *Options) RepositoryWithManifests(ctx context.Context, ref TypedImageReference) (distribution.Repository, distribution.ManifestService, error) { switch ref.Type { case DestinationRegistry: repo, err := o.RegistryContext.Repository(ctx, ref.Ref.DockerClientDefaults().RegistryURL(), ref.Ref.RepositoryName(), o.Insecure) if err != nil { return nil, nil, err } - manifests, err := repo.Manifests(context.TODO()) + manifests, err := repo.Manifests(ctx) if err != nil { return nil, nil, fmt.Errorf("unable to get local manifest service: %v", err) } return repo, manifests, nil + case DestinationFile: driver := &fileDriver{ BaseDir: o.FileDir, } - repo, err := driver.Repository(ctx, ref.Ref.DockerClientDefaults().RegistryURL(), ref.Ref.RepositoryName(), o.Insecure) - if err != nil { - return nil, nil, err - } - return repo, nil, nil + return driver.Repository(ctx, ref.Ref.DockerClientDefaults().RegistryURL(), ref.Ref.RepositoryName(), o.Insecure) case DestinationS3: driver := &s3Driver{ Creds: o.RegistryContext.Credentials, CopyFrom: o.AttemptS3BucketCopy, } url := ref.Ref.DockerClientDefaults().RegistryURL() - repo, err := driver.Repository(ctx, url, ref.Ref.RepositoryName(), o.Insecure) - if err != nil { - return nil, nil, err - } - return repo, nil, nil + return driver.Repository(ctx, url, ref.Ref.RepositoryName(), o.Insecure) default: return nil, nil, fmt.Errorf("unrecognized image reference type %s", ref.Type) } } -type registryContext struct { - *registryclient.Context -} - -func (c *registryContext) Repository(ctx context.Context, registry *url.URL, repoName string, insecure bool) (distribution.Repository, error) { - var repo distribution.Repository - var err error - if len(c.ImageSources) == 0 { - repo, err = c.Repository(ctx, registry, repoName, insecure) - if err != nil { - return nil, err - } - } - for _, ics := range c.ImageSources { - repo, err = c.Repository(ctx, ics.RegistryURL(), ics.RepositoryName(), insecure) - if err != nil { - continue - } - - // it would be nice to simply return ManifestService here, as we'll need it, but this will not satifsy library-go's RepositoryRetriever interface - _, err := repo.Manifests(context.TODO()) - if err != nil { - err = fmt.Errorf("unable to get local manifest service: %v", err) - continue - } - break - } - return repo, nil -} - // ExpandWildcard expands the provided typed reference (which is known to have an expansion) // to a set of explicit image references. func (o *Options) ExpandWildcard(ref TypedImageReference) ([]TypedImageReference, error) { @@ -96,7 +57,7 @@ func (o *Options) ExpandWildcard(ref TypedImageReference) ([]TypedImageReference } // lookup tags that match the search - repo, _, err := o.Repository(context.Background(), ref) + repo, _, err := o.RepositoryWithManifests(context.Background(), ref) if err != nil { return nil, err } diff --git a/pkg/cli/image/imagesource/s3.go b/pkg/cli/image/imagesource/s3.go index eaee32512b..a269b94e78 100644 --- a/pkg/cli/image/imagesource/s3.go +++ b/pkg/cli/image/imagesource/s3.go @@ -106,23 +106,23 @@ func (d *s3Driver) newObject(server *url.URL, region string, insecure bool, secu return s3obj, nil } -func (d *s3Driver) Repository(ctx context.Context, server *url.URL, repoName string, insecure bool) (distribution.Repository, error) { +func (d *s3Driver) Repository(ctx context.Context, server *url.URL, repoName string, insecure bool) (distribution.Repository, distribution.ManifestService, error) { parts := strings.SplitN(repoName, "/", 3) if len(parts) < 3 { - return nil, fmt.Errorf("you must pass a three segment repository name for s3 uploads, where the first segment is the region and the second segment is the bucket") + return nil, nil, fmt.Errorf("you must pass a three segment repository name for s3 uploads, where the first segment is the region and the second segment is the bucket") } s3obj, err := d.newObject(server, parts[0], insecure, &url.URL{Scheme: server.Scheme, Host: server.Host, Path: "/" + repoName}) if err != nil { - return nil, err + return nil, nil, err } ref, err := reference.Parse(parts[2]) if err != nil { - return nil, err + return nil, nil, err } named, ok := ref.(reference.Named) if !ok { - return nil, fmt.Errorf("%s is not a valid repository name", parts[2]) + return nil, nil, fmt.Errorf("%s is not a valid repository name", parts[2]) } repo := &s3Repository{ @@ -132,7 +132,11 @@ func (d *s3Driver) Repository(ctx context.Context, server *url.URL, repoName str repoName: named, copyFrom: d.CopyFrom, } - return repo, nil + manifests, err := repo.Manifests(context.TODO()) + if err != nil { + return nil, nil, fmt.Errorf("unable to get local manifest service: %v", err) + } + return repo, manifests, nil } type s3Repository struct { diff --git a/pkg/cli/image/info/info.go b/pkg/cli/image/info/info.go index ed7d0776f5..670e7878e0 100644 --- a/pkg/cli/image/info/info.go +++ b/pkg/cli/image/info/info.go @@ -38,7 +38,7 @@ func NewInfoOptions(streams genericclioptions.IOStreams) *InfoOptions { } } -func NewInfo(streams genericclioptions.IOStreams) *cobra.Command { +func NewInfo(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewInfoOptions(streams) cmd := &cobra.Command{ Use: "info IMAGE [...]", @@ -68,7 +68,7 @@ func NewInfo(streams genericclioptions.IOStreams) *cobra.Command { `), Run: func(cmd *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(cmd, args)) + kcmdutil.CheckErr(o.Complete(f, cmd, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -94,11 +94,14 @@ type InfoOptions struct { Output string } -func (o *InfoOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *InfoOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if len(args) < 1 { return fmt.Errorf("info expects at least one argument, an image pull spec") } o.Images = args + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } return nil } @@ -110,7 +113,7 @@ func (o *InfoOptions) Run() error { if len(o.Images) == 0 { return fmt.Errorf("must specify one or more images as arguments") } - regContext, err := o.SecurityOptions.Context(imagereference.DockerImageReference{}) + regContext, err := o.SecurityOptions.Context() if err != nil { return err } @@ -364,15 +367,10 @@ type ImageRetriever struct { func (o *ImageRetriever) Run() error { ctx := context.Background() - fromContext, err := o.SecurityOptions.Context(imagereference.DockerImageReference{}) + fromContext, err := o.SecurityOptions.Context() if err != nil { return err } - fromOptions := &imagesource.Options{ - FileDir: o.FileDir, - Insecure: o.SecurityOptions.Insecure, - RegistryContext: fromContext, - } callbackFn := o.ImageMetadataCallback if callbackFn == nil { @@ -388,10 +386,12 @@ func (o *ImageRetriever) Run() error { name := key from := o.Image[key] q.Try(func() error { - repo, manifests, err := fromOptions.Repository(ctx, from) + var newImageRef imagereference.DockerImageReference + newImageRef, repo, manifests, err := o.SecurityOptions.PreferredImageSource(from.Ref, fromContext) if err != nil { return callbackFn(name, nil, fmt.Errorf("unable to connect to image repository %s: %v", from, err)) } + from.Ref = newImageRef allManifests, manifestList, listDigest, err := imagemanifest.AllManifests(ctx, from.Ref, manifests, repo) if err != nil { diff --git a/pkg/cli/image/manifest/.security_test.go.swp b/pkg/cli/image/manifest/.security_test.go.swp deleted file mode 100644 index 748383ff318e36bfdb81542fb7ad8f3b6e6b71e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI3Uu+yl9ml6>2wa;&pn?|!Os=##C-$ygNE;mlDYD~Y(X^p<(^ge+*L!p4jrVq! z-8rX+V-gS#NW7rJ3lb7kDr!}s(1%tPR4NdaKT<(z#lIjyAByM;h^Po65fJeG&7Zx! z^Id93sR-4s^zrTO&dkny=QlII-`sj>^61=QzTdf-;n>C4>1R(b&beP;KdaorvbgPq zq3F`BiydL?d*O1NNk^SEkS&OYWcaKt(Dmbna9i>5I9+kG)OTBfT<$D6e%yBBL`2zg z(3EcMyU9w+^~1mo+FnbzZ7&L%B9m?={7xFk?xMsurxg!*R(YvZpj4osKu0zw_Fu<7 zbo10CUA>{Yn_vH~y9)`*(^7#_fl`4|fl`4|fl`4|fl`4|fj7GXGTFvXqLJH7L)Xmb zRRf<_o9~}ApC<Q6%z~ZZpVu(Xc;2cHACfrH?EU}vMUT_sS|5nES4E_MV z1HKB5fkkjPxE;)Z1K?J0J^0rS=mow9V$cL11-rp(Z(;0J@H_Ae@B(-qJO>^D$H0HK zGj;)-2WP-%zyf$L_&qjf&VpZpZ-EEEtzaAY3HtR6cn*9GEQ0&MKJb1({r+yye^lsL zA7|>ba);LzbzV(HD7;Km&E-3njtO5L7SgNpHvXJ8ysi_(ZsKK`Xy{i;s)ej|K5z`pQU9>B zPb)y}nhioIQ#|a|njBL)^;%y{wG}0F6}1+{GU&V-#YPb&QWMqLqAGYLN2H>K{zoy@KN!(||a)TtVVw%Al;xTo#42TuE$OTpj0CFN)=|NOKkQ?}cQW zlC#|jWsrozC8Nn+&(f+|X(V}u)3$mC=EWfou@_ZiEnnK~{lUdS+YH?`D^*{DzPq$O z_ubt>6Yv0PKwozHVjn;YM8@j}bbUL%zXyDeypb%;xokQdwG;38%#Bs`xrvdVE8FMN|<>9oh)%lI9 zn6r@x8G|OB@ZAS^6ohz4g}xo&LEMo+=-h)hOa(3QwSgy&qa{mSsjrWX^T{#xAX6rI{8X>h}s)C-bIJHh*gTW{sX-3jaM)GuDFG1RSSS%F@rRqexJUK^-~p>dO9%Yy?)+hP*)} z;EvMU-?)&ulnCZih2>DZkcuXVe8HIVC_?3d!)7v}ltF(1ub3nL z7*U$ONrl;L^h!F?XF|HG+T;;W$Zmqac@k^vL!LI`IcVw7?8P>=;uoX}E?ZgFD z#wyC7F!56_D;A(II zvHgqSY49a*3ao-Q5a3hbUhomH9sB@s{kAoQ0!FKQ(V*2lbr@$$20<3{W@ImlS@G@ff--2I(C&7K-Hn0oq1ZSb& z0yqK=1JZL6lt-yRsX(c~n?`~4(TDct$VDpn;|Cj!;Z8np#S-53jjQAQxQEr3P8vb8 zsjk6K^Fp&Dn8Bqp`+0YXCca_ab7qjTHY&*&91HU9cbnzVnGqs61h+h_^_SEufPxIf z!~J!)TRPl02*&55r~xqs)jf|THU<-wUg5b<>N$`@xrR2-QO*gPHO}F5k-s%^OO$*t zYL`dO?M>b&S+g~2gldTTQ|JohQL!a&rg2+^Lql6Kq$w&g!eeb6`&*&5Yy*+aqF5C$ zN$IF|nQjP0RHI8K3?1ViJdMrn` z8wYV%PYrEf&%pA|Qf1{uIY;makyndq(S#8!_KcJICG{eBSGN78lMk7~m5Hr-X`eYG zm>hV}Ozf|SbdEe-7=~_$)ynwo6l74)ip#c3XwzU?`K7ZFRHO*IHpXk#A+`Rfw&amU zHiw<~wCVXmsSu}nSeR!}197VU^-_ffrc+fUqZ(~{*jBd}mT#qlHpJ!H?N_6OLUdGP zKr-DoR(AA_=^7~uor170qb^Y6gCeD&+bV?(PQ5#(9@QUeA(+bdI`bgkM9iDqwzUyp zf59erG&@2zX0xBP+fUtWw}#C2ovPq~sTErBI{SK?YNXz{(7z9eH-2ZjZ)@Z-ZIJdi z!EW>RwnazmGn0^mW5^kimR1lPmv2w{f3*{-ek##|>7SQGc1b_E_4T9q1y38%=6-m) q#Ia>|we9A%{VeQu=a;aOsF^@@+`txie}#L=E@@kSpewH!yZbLXwdwW% diff --git a/pkg/cli/image/manifest/manifest.go b/pkg/cli/image/manifest/manifest.go index 7507aefac6..8d0ebf862d 100644 --- a/pkg/cli/image/manifest/manifest.go +++ b/pkg/cli/image/manifest/manifest.go @@ -164,6 +164,7 @@ func AllManifests(ctx context.Context, from imagereference.DockerImageReference, if err != nil { return nil, nil, "", err } + return ManifestsFromList(ctx, srcDigest, srcManifest, manifests, from) } diff --git a/pkg/cli/image/manifest/security.go b/pkg/cli/image/manifest/security.go index e7f3d08d01..79d8f2efc1 100644 --- a/pkg/cli/image/manifest/security.go +++ b/pkg/cli/image/manifest/security.go @@ -8,6 +8,8 @@ import ( "github.com/spf13/pflag" + "github.com/docker/distribution" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" @@ -19,16 +21,21 @@ import ( operatorv1alpha1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1alpha1" imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" + "github.com/openshift/oc/pkg/cli/image/imagesource" "github.com/openshift/oc/pkg/cli/image/manifest/dockercredentials" ) type SecurityOptions struct { - RegistryConfig string - Insecure bool - SkipVerification bool - ImageContentSourcePolicyFile string - ImageContentSourcePolicyList []operatorv1alpha1.ImageContentSourcePolicy - ICSPClientFn func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) + RegistryConfig string + Insecure bool + SkipVerification bool + // ImageContentSourcePolicyFile to look up alternative sources + ICSPFile string + LookupClusterICSP bool + ICSPList []operatorv1alpha1.ImageContentSourcePolicy + ICSPClientFn func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) + TryAlternativeSources bool + FileDir string CachedContext *registryclient.Context } @@ -37,55 +44,34 @@ func (o *SecurityOptions) Bind(flags *pflag.FlagSet) { flags.StringVarP(&o.RegistryConfig, "registry-config", "a", o.RegistryConfig, "Path to your registry credentials (defaults to ~/.docker/config.json)") flags.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP") flags.BoolVar(&o.SkipVerification, "skip-verification", o.SkipVerification, "Skip verifying the integrity of the retrieved content. This is not recommended, but may be necessary when importing images from older image registries. Only bypass verification if the registry is known to be trustworthy.") - flags.StringVar(&o.ImageContentSourcePolicyFile, "icsp-file", o.ImageContentSourcePolicyFile, "Path to an ImageContentSourcePolicy file. If set, data from this file will be used to set source release image.") + flags.BoolVar(&o.LookupClusterICSP, "lookup-cluster-icsp", o.LookupClusterICSP, "If set to true, look for alternative image sources from ImageContentSourcePolicy objects in cluster, honor the ordering of those sources, and fail if an ImageContentSourcePolicy is not found in cluster. Cannot be set to true with --icsp-file") + flags.StringVar(&o.ICSPFile, "icsp-file", o.ICSPFile, "Path to an ImageContentSourcePolicy file. If set, data from this file will be used to set alternative image sources. Cannot be set together with --lookup-cluster-icsp=true.") } // ReferentialHTTPClient returns an http.Client that is appropriate for accessing // blobs referenced outside of the registry (due to the present of the URLs attribute // in the manifest reference for a layer). func (o *SecurityOptions) ReferentialHTTPClient() (*http.Client, error) { - ctx, err := o.Context(imagereference.DockerImageReference{}) + regContext, err := o.Context() if err != nil { return nil, err } client := &http.Client{} if o.Insecure { - client.Transport = ctx.InsecureTransport + client.Transport = regContext.InsecureTransport } else { - client.Transport = ctx.Transport + client.Transport = regContext.Transport } return client, nil } -func (o *SecurityOptions) AddImageSourcePoliciesFromFile(image string) error { - if len(image) == 0 { - return fmt.Errorf("expected image to find image sources") - } - icspData, err := ioutil.ReadFile(o.ImageContentSourcePolicyFile) - if err != nil { - return fmt.Errorf("unable to read ImageContentSourceFile %s: %v", o.ImageContentSourcePolicyFile, err) +func (o *SecurityOptions) Complete(f kcmdutil.Factory) error { + if o.LookupClusterICSP && len(o.ICSPFile) > 0 { + return fmt.Errorf("cannot set both --lookup-cluster-icsp=true and --icsp-file") } - if len(icspData) == 0 { - return fmt.Errorf("no data found in ImageContentSourceFile %s", o.ImageContentSourcePolicyFile) - } - icspObj, err := runtime.Decode(operatorv1alpha1scheme.Codecs.UniversalDeserializer(), icspData) - if err != nil { - return fmt.Errorf("error decoding ImageContentSourcePolicy from %s: %v", o.ImageContentSourcePolicyFile, err) - } - var icsp *operatorv1alpha1.ImageContentSourcePolicy - var ok bool - if icsp, ok = icspObj.(*operatorv1alpha1.ImageContentSourcePolicy); !ok { - return fmt.Errorf("could not decode ImageContentSourcePolicy from %s", o.ImageContentSourcePolicyFile) - } - o.ImageContentSourcePolicyList = append(o.ImageContentSourcePolicyList, *icsp) - return nil - -} - -func (o *SecurityOptions) Complete(f kcmdutil.Factory, image string) error { o.ICSPClientFn = func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) { // If ImageContentSourceFile is given, only add ImageContentSource from file, don't search cluster ICSP - if len(o.ImageContentSourcePolicyFile) != 0 { + if len(o.ICSPFile) != 0 { return nil, nil } restConfig, err := f.ToRESTConfig() @@ -107,36 +93,11 @@ func (o *SecurityOptions) Complete(f kcmdutil.Factory, image string) error { return nil } -func (o *SecurityOptions) AddICSPsFromCluster() error { - icspClient, err := o.ICSPClientFn() - if err != nil { - return err - } - if icspClient != nil { - o.GetICSPs(icspClient) - } - return nil -} - -// GetICSPs will lookup ICSPs from cluster. Since it's not a hard requirement to find ICSPs from cluster, GetICSPs logs errors rather than returning errors. -func (o *SecurityOptions) GetICSPs(icspClient operatorv1alpha1client.ImageContentSourcePolicyInterface) { - icsps, err := icspClient.List(context.TODO(), metav1.ListOptions{}) - if err != nil { - // may or may not have access to ICSPs in cluster - // don't error if can't access ICSPs - klog.V(4).Infof("did not access any ImageContentSourcePolicies in cluster: %v", err) - } - if len(icsps.Items) == 0 { - klog.V(4).Info("no ImageContentSourcePolicies found in cluster") - } - o.ImageContentSourcePolicyList = append(o.ImageContentSourcePolicyList, icsps.Items...) -} - -func (o *SecurityOptions) Context(image imagereference.DockerImageReference) (*registryclient.Context, error) { +func (o *SecurityOptions) Context() (*registryclient.Context, error) { if o.CachedContext != nil { return o.CachedContext, nil } - context, err := o.NewContext(image) + context, err := o.NewContext() if err == nil { o.CachedContext = context o.CachedContext.Retries = 3 @@ -144,7 +105,7 @@ func (o *SecurityOptions) Context(image imagereference.DockerImageReference) (*r return context, err } -func (o *SecurityOptions) NewContext(image imagereference.DockerImageReference) (*registryclient.Context, error) { +func (o *SecurityOptions) NewContext() (*registryclient.Context, error) { rt, err := rest.TransportFor(&rest.Config{}) if err != nil { return nil, err @@ -162,25 +123,63 @@ func (o *SecurityOptions) NewContext(image imagereference.DockerImageReference) } context := registryclient.NewContext(rt, insecureRT).WithCredentials(creds) context.DisableDigestVerification = o.SkipVerification - if len(image.String()) > 0 { - context.AlternativeSources = &addAlternativeImageSources{} - altSources, err := context.AlternativeSources.AddImageSources(image, o.ImageContentSourcePolicyList) + return context, nil +} + +// AddICSPsFromCluster will lookup ICSPs from cluster. Since it's not a hard requirement to find ICSPs from cluster, logs errors rather than returning errors. +func (a *SecurityOptions) AddICSPsFromCluster() error { + icspClient, err := a.ICSPClientFn() + if err != nil { + return err + } + if a.LookupClusterICSP && icspClient == nil { + return fmt.Errorf("flag --lookup-cluster-icsp was set to true, but no method was set to find ImageContentSourcePolicy objects in cluster") + } + if icspClient != nil { + icsps, err := icspClient.List(context.TODO(), metav1.ListOptions{}) if err != nil { - return nil, err + if a.LookupClusterICSP { + return fmt.Errorf("--lookup-cluster-icsp was set to true, but did not access ImageContentSourcePolicy objects in cluster: %v", err) + } + // may or may not have access to ICSPs in cluster + // don't error if can't access ICSPs + klog.V(4).Infof("did not access any ImageContentSourcePolicies in cluster: %v", err) + } + if len(icsps.Items) == 0 { + if a.LookupClusterICSP { + return fmt.Errorf("--lookup-cluster-icsp was set to true, but no ImageContentSourcePolicy objects found in cluster: %v", err) + } + klog.V(4).Info("no ImageContentSourcePolicies found in cluster") } - context.ImageSources = altSources + a.ICSPList = append(a.ICSPList, icsps.Items...) } - return context, nil + return nil } -type addAlternativeImageSources struct{} - -func (a *addAlternativeImageSources) AddImageSources(imageRef imagereference.DockerImageReference, icspList []operatorv1alpha1.ImageContentSourcePolicy) ([]imagereference.DockerImageReference, error) { - if len(icspList) == 0 { - return nil, nil +func (a *SecurityOptions) AddImageSourcePoliciesFromFile() error { + icspData, err := ioutil.ReadFile(a.ICSPFile) + if err != nil { + return fmt.Errorf("unable to read ImageContentSourceFile %s: %v", a.ICSPFile, err) + } + if len(icspData) == 0 { + return fmt.Errorf("no data found in ImageContentSourceFile %s", a.ICSPFile) } + icspObj, err := runtime.Decode(operatorv1alpha1scheme.Codecs.UniversalDeserializer(), icspData) + if err != nil { + return fmt.Errorf("error decoding ImageContentSourcePolicy from %s: %v", a.ICSPFile, err) + } + var icsp *operatorv1alpha1.ImageContentSourcePolicy + var ok bool + if icsp, ok = icspObj.(*operatorv1alpha1.ImageContentSourcePolicy); !ok { + return fmt.Errorf("could not decode ImageContentSourcePolicy from %s", a.ICSPFile) + } + a.ICSPList = append(a.ICSPList, *icsp) + return nil +} + +func (a *SecurityOptions) AddImageSources(imageRef imagereference.DockerImageReference) ([]imagereference.DockerImageReference, error) { var imageSources []imagereference.DockerImageReference - for _, icsp := range icspList { + for _, icsp := range a.ICSPList { repoDigestMirrors := icsp.Spec.RepositoryDigestMirrors var sourceMatches bool for _, rdm := range repoDigestMirrors { @@ -204,11 +203,6 @@ func (a *addAlternativeImageSources) AddImageSources(imageRef imagereference.Doc } } } - // make sure at least 1 imagesource - // ie, make sure the image passed is included in image sources - if len(imageSources) == 0 { - imageSources = append(imageSources, imageRef.AsRepository()) - } uniqueMirrors := make([]imagereference.DockerImageReference, 0, len(imageSources)) uniqueMap := make(map[imagereference.DockerImageReference]bool) for _, imageSourceMirror := range imageSources { @@ -217,6 +211,118 @@ func (a *addAlternativeImageSources) AddImageSources(imageRef imagereference.Doc uniqueMirrors = append(uniqueMirrors, imageSourceMirror) } } + // make sure at least 1 imagesource + // ie, make sure the image passed is included in image sources + // this is so the user-given image ref will be tried + if len(imageSources) == 0 { + imageSources = append(imageSources, imageRef.AsRepository()) + return imageSources, nil + } klog.V(2).Infof("Found sources: %v for image: %v", uniqueMirrors, imageRef) return uniqueMirrors, nil } + +func (s *SecurityOptions) PreferredImageSource(image imagereference.DockerImageReference, regContext *registryclient.Context) (imagereference.DockerImageReference, distribution.Repository, distribution.ManifestService, error) { + var ( + repo distribution.Repository + replacedImage imagereference.DockerImageReference + manifests distribution.ManifestService + err error + ) + ctx := context.TODO() + typedImageRef := imagesource.TypedImageReference{Ref: image, Type: imagesource.DestinationRegistry} + + sourceOpts := &imagesource.Options{ + FileDir: s.FileDir, + Insecure: s.Insecure, + RegistryContext: regContext, + } + if len(s.ICSPFile) == 0 && !s.LookupClusterICSP && !s.TryAlternativeSources { + repo, manifests, err = sourceOpts.RepositoryWithManifests(ctx, typedImageRef) + if err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + if repo == nil || manifests == nil { + return image, nil, nil, fmt.Errorf("unable to retrieve image manifests for %v", image.String()) + } + return image, repo, manifests, nil + } + + var altSources []imagereference.DockerImageReference + // always error if given an icsp file and don't successfully connect to image repository + if len(s.ICSPFile) > 0 { + if err = s.AddImageSourcePoliciesFromFile(); err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + altSources, err = s.AddImageSources(image) + if err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + for _, icsRef := range altSources { + replacedImage = replaceImage(icsRef, image) + // if not successful, error will be handled below + if repo, manifests, err = sourceOpts.RepositoryWithManifests(context.TODO(), imagesource.TypedImageReference{Ref: replacedImage, Type: imagesource.DestinationRegistry}); err == nil { + if repo == nil || manifests == nil { + return image, nil, nil, fmt.Errorf("unable to retrieve image manifests for %v", image.String()) + } + return replacedImage, repo, manifests, nil + } + } + } + + // always error if user passed flag to look in cluster and didn't connect to image repository + if s.LookupClusterICSP { + // now try to look for ICSPs from cluster + if s.ICSPClientFn == nil { + return imagereference.DockerImageReference{}, nil, nil, fmt.Errorf("unable to find ImageContentSourcePolicy object from cluster") + } + if err = s.AddICSPsFromCluster(); err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + altSources, err = s.AddImageSources(image) + if err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + for _, icsRef := range altSources { + replacedImage = replaceImage(icsRef, image) + repo, manifests, err = sourceOpts.RepositoryWithManifests(context.TODO(), imagesource.TypedImageReference{Ref: replacedImage, Type: imagesource.DestinationRegistry}) + if err == nil && repo != nil && manifests != nil { + return replacedImage, repo, manifests, nil + } + } + } + + // now implicitly try other sources, only if TryAlternativeSources set + if s.TryAlternativeSources { + if s.ICSPClientFn == nil { + repo, manifests, err = sourceOpts.RepositoryWithManifests(context.TODO(), typedImageRef) + if err == nil && repo != nil && manifests != nil { + return typedImageRef.Ref, repo, manifests, nil + } + } + if err = s.AddICSPsFromCluster(); err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + altSources, err = s.AddImageSources(image) + if err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + for _, icsRef := range altSources { + replacedImage = replaceImage(icsRef, image) + repo, manifests, err = sourceOpts.RepositoryWithManifests(context.TODO(), imagesource.TypedImageReference{Ref: replacedImage, Type: imagesource.DestinationRegistry}) + if err == nil && repo != nil && manifests != nil { + return replacedImage, repo, manifests, nil + } + } + } + if err != nil { + return imagereference.DockerImageReference{}, nil, nil, fmt.Errorf("unable to connect to imagerepository %s: %v", image.String(), err) + } + return replacedImage, repo, manifests, err +} + +func replaceImage(icsRef imagereference.DockerImageReference, image imagereference.DockerImageReference) imagereference.DockerImageReference { + icsRef.ID = image.ID + icsRef.Tag = image.Tag + return icsRef +} diff --git a/pkg/cli/image/manifest/security_test.go b/pkg/cli/image/manifest/security_test.go index 26f20b57af..5ca97dae51 100644 --- a/pkg/cli/image/manifest/security_test.go +++ b/pkg/cli/image/manifest/security_test.go @@ -7,18 +7,15 @@ import ( "github.com/google/go-cmp/cmp" operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1" - operatorv1alpha1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1alpha1" imagereference "github.com/openshift/library-go/pkg/image/reference" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func securityOpts(icspList []operatorv1alpha1.ImageContentSourcePolicy, icspFile string) *SecurityOptions { +func lookupAlternate(icspList []operatorv1alpha1.ImageContentSourcePolicy, icspFile string) *SecurityOptions { return &SecurityOptions{ - Insecure: true, - SkipVerification: true, - ImageContentSourcePolicyFile: icspFile, - ImageContentSourcePolicyList: icspList, - CachedContext: nil, + ICSPFile: icspFile, + ICSPList: icspList, + LookupClusterICSP: false, } } @@ -165,7 +162,7 @@ func TestAlternativeImageSources(t *testing.T) { icspList: nil, icspFile: "", image: "quay.io/ocp-test/release:4.5", - imageSourcesExpected: []string{}, + imageSourcesExpected: []string{"quay.io/ocp-test/release"}, }, } for _, tt := range tests { @@ -173,10 +170,6 @@ func TestAlternativeImageSources(t *testing.T) { if err != nil { t.Errorf("parsing image reference error = %v", err) } - secOpts := securityOpts(tt.icspList, tt.icspFile) - secOpts.ICSPClientFn = func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) { - return nil, nil - } var expectedRefs []imagereference.DockerImageReference for _, expected := range tt.imageSourcesExpected { expectedRef, err := imagereference.Parse(expected) @@ -186,14 +179,14 @@ func TestAlternativeImageSources(t *testing.T) { expectedRefs = append(expectedRefs, expectedRef) } + a := lookupAlternate(tt.icspList, tt.icspFile) if len(tt.icspFile) > 0 { - err := secOpts.AddImageSourcePoliciesFromFile(tt.image) + err := a.AddImageSourcePoliciesFromFile() if err != nil { t.Errorf("add ICSP from file error = %v", err) } } - a := &addAlternativeImageSources{} - altSources, err := a.AddImageSources(imageRef, secOpts.ImageContentSourcePolicyList) + altSources, err := a.AddImageSources(imageRef) if err != nil { t.Errorf("registry client Context error = %v", err) } diff --git a/pkg/cli/image/mirror/mirror.go b/pkg/cli/image/mirror/mirror.go index 3ee06a676d..fa90f8bc4c 100644 --- a/pkg/cli/image/mirror/mirror.go +++ b/pkg/cli/image/mirror/mirror.go @@ -26,7 +26,6 @@ import ( kcmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" - imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" "github.com/openshift/oc/pkg/cli/image/imagesource" imagemanifest "github.com/openshift/oc/pkg/cli/image/manifest" @@ -143,7 +142,7 @@ func NewMirrorImageOptions(streams genericclioptions.IOStreams) *MirrorImageOpti } // NewCommandMirrorImage copies images from one location to another. -func NewCmdMirrorImage(streams genericclioptions.IOStreams) *cobra.Command { +func NewCmdMirrorImage(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewMirrorImageOptions(streams) cmd := &cobra.Command{ @@ -152,16 +151,16 @@ func NewCmdMirrorImage(streams genericclioptions.IOStreams) *cobra.Command { Long: mirrorDesc, Example: mirrorExample, Run: func(c *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(c, args)) + kcmdutil.CheckErr(o.Complete(f, c, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, } flag := cmd.Flags() - o.SecurityOptions.Bind(flag) o.FilterOptions.Bind(flag) o.ParallelOptions.Bind(flag) + o.SecurityOptions.Bind(flag) flag.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Print the actions that would be taken and exit without writing to the destinations.") flag.BoolVar(&o.ContinueOnError, "continue-on-error", o.ContinueOnError, "If an error occurs, keep going and attempt to mirror as much as possible.") @@ -179,18 +178,17 @@ func NewCmdMirrorImage(streams genericclioptions.IOStreams) *cobra.Command { return cmd } -func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *MirrorImageOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { return err } - if o.FilterOptions.IsWildcardFilter() { - o.KeepManifestList = true + if err := o.SecurityOptions.Complete(f); err != nil { + return err } - registryContext, err := o.SecurityOptions.Context(imagereference.DockerImageReference{}) - if err != nil { - return err + if o.FilterOptions.IsWildcardFilter() { + o.KeepManifestList = true } dir := o.FileDir @@ -198,6 +196,11 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { dir = o.FromFileDir } + registryContext, err := o.SecurityOptions.Context() + if err != nil { + return err + } + opts := &imagesource.Options{ FileDir: dir, Insecure: o.SecurityOptions.Insecure, @@ -232,7 +235,7 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { return nil } -func (o *MirrorImageOptions) Repository(ctx context.Context, regContext *registryclient.Context, ref imagesource.TypedImageReference, source bool) (distribution.Repository, distribution.ManifestService, error) { +func (o *MirrorImageOptions) RepositoryWithManifests(ctx context.Context, regContext *registryclient.Context, ref imagesource.TypedImageReference, source bool) (distribution.Repository, distribution.ManifestService, error) { dir := o.FileDir if len(o.FromFileDir) > 0 && source { dir = o.FromFileDir @@ -245,7 +248,7 @@ func (o *MirrorImageOptions) Repository(ctx context.Context, regContext *registr AttemptS3BucketCopy: o.AttemptS3BucketCopy, RegistryContext: regContext, } - return opts.Repository(ctx, ref) + return opts.RepositoryWithManifests(ctx, ref) } func (o *MirrorImageOptions) Validate() error { @@ -433,12 +436,12 @@ type contextKey struct { func (o *MirrorImageOptions) plan() (*plan, error) { ctx := apirequest.NewContext() - context, err := o.SecurityOptions.Context(imagereference.DockerImageReference{}) + regContext, err := o.SecurityOptions.Context() if err != nil { return nil, err } - fromContext := context.Copy() - toContext := context.Copy().WithActions("pull", "push") + fromContext := regContext.Copy() + toContext := regContext.Copy().WithActions("pull", "push") toContexts := make(map[contextKey]*registryclient.Context) tree := buildTargetTree(o.Mappings) @@ -466,11 +469,12 @@ func (o *MirrorImageOptions) plan() (*plan, error) { for name := range tree { src := tree[name] q.Queue(func(_ workqueue.Work) { - srcRepo, manifests, err := o.Repository(ctx, fromContext, src.ref, true) + newImageRef, srcRepo, manifests, err := o.SecurityOptions.PreferredImageSource(src.ref.Ref, fromContext) if err != nil { plan.AddError(retrieverError{err: fmt.Errorf("unable to connect to %s: %v", src.ref, err), src: src.ref}) return } + src.ref.Ref = newImageRef rq := registryWorkers[name.registry] rq.Batch(func(w workqueue.Work) { @@ -501,6 +505,7 @@ func (o *MirrorImageOptions) plan() (*plan, error) { rq.Queue(func(w workqueue.Work) { for key := range src.digests { srcDigestString, pushTargets := key, src.digests[key] + w.Parallel(func() { // load the manifest srcDigest := godigest.Digest(srcDigestString) @@ -531,17 +536,19 @@ func (o *MirrorImageOptions) plan() (*plan, error) { } for _, dst := range pushTargets { + var toRepo distribution.Repository var toManifests distribution.ManifestService var err error if o.DryRun { toRepo, err = imagesource.NewDryRun(dst.ref) } else { - toRepo, toManifests, err = o.Repository(ctx, toContexts[contextKeyForReference(dst.ref)], dst.ref, false) - } - if err != nil { - plan.AddError(retrieverError{src: src.ref, dst: dst.ref, err: fmt.Errorf("unable to connect to %s: %v", dst.ref, err)}) - continue + newImageRef, toRepo, toManifests, err = o.SecurityOptions.PreferredImageSource(dst.ref.Ref, toContexts[contextKeyForReference(dst.ref)]) + if err != nil { + plan.AddError(retrieverError{src: src.ref, dst: dst.ref, err: fmt.Errorf("unable to connect to %s: %v", dst.ref, err)}) + continue + } + dst.ref.Ref = newImageRef } canonicalTo := toRepo.Named()