diff --git a/contrib/completions/bash/oc b/contrib/completions/bash/oc index a25ada03b3..4b8c2fac84 100644 --- a/contrib/completions/bash/oc +++ b/contrib/completions/bash/oc @@ -576,8 +576,13 @@ _oc_adm_catalog_build() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--manifest-dir=") two_word_flags+=("--manifest-dir") local_nonpersistent_flags+=("--manifest-dir=") @@ -665,11 +670,16 @@ _oc_adm_catalog_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--icsp-scope=") two_word_flags+=("--icsp-scope") local_nonpersistent_flags+=("--icsp-scope=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--manifests-only") local_nonpersistent_flags+=("--manifests-only") flags+=("--max-components=") @@ -5444,8 +5454,13 @@ _oc_adm_release_extract() flags+=("--git=") two_word_flags+=("--git") local_nonpersistent_flags+=("--git=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5545,6 +5560,9 @@ _oc_adm_release_info() flags+=("--dir=") two_word_flags+=("--dir") local_nonpersistent_flags+=("--dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image-for=") two_word_flags+=("--image-for") local_nonpersistent_flags+=("--image-for=") @@ -5552,6 +5570,8 @@ _oc_adm_release_info() local_nonpersistent_flags+=("--include-images") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5646,8 +5666,13 @@ _oc_adm_release_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5657,6 +5682,9 @@ _oc_adm_release_mirror() two_word_flags+=("--registry-config") two_word_flags+=("-a") local_nonpersistent_flags+=("--registry-config=") + flags+=("--release-image-icsp-to-dir=") + two_word_flags+=("--release-image-icsp-to-dir") + local_nonpersistent_flags+=("--release-image-icsp-to-dir=") flags+=("--release-image-signature-to-dir=") two_word_flags+=("--release-image-signature-to-dir") local_nonpersistent_flags+=("--release-image-signature-to-dir=") @@ -5765,11 +5793,16 @@ _oc_adm_release_new() flags+=("--from-release=") two_word_flags+=("--from-release") local_nonpersistent_flags+=("--from-release=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--include=") two_word_flags+=("--include") local_nonpersistent_flags+=("--include=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--mapping-file=") two_word_flags+=("--mapping-file") local_nonpersistent_flags+=("--mapping-file=") @@ -13408,11 +13441,16 @@ _oc_image_append() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image=") two_word_flags+=("--image") local_nonpersistent_flags+=("--image=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -13501,8 +13539,13 @@ _oc_image_extract() flags+=("--filter-by-os=") two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--only-files") local_nonpersistent_flags+=("--only-files") flags+=("--path=") @@ -13581,8 +13624,13 @@ _oc_image_info() flags+=("--filter-by-os=") two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--output=") two_word_flags+=("--output") two_word_flags+=("-o") @@ -13670,10 +13718,15 @@ _oc_image_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--keep-manifest-list") local_nonpersistent_flags+=("--keep-manifest-list") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") diff --git a/contrib/completions/zsh/oc b/contrib/completions/zsh/oc index fc99cb14cd..032aa204a6 100644 --- a/contrib/completions/zsh/oc +++ b/contrib/completions/zsh/oc @@ -676,8 +676,13 @@ _oc_adm_catalog_build() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--manifest-dir=") two_word_flags+=("--manifest-dir") local_nonpersistent_flags+=("--manifest-dir=") @@ -765,11 +770,16 @@ _oc_adm_catalog_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--icsp-scope=") two_word_flags+=("--icsp-scope") local_nonpersistent_flags+=("--icsp-scope=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--manifests-only") local_nonpersistent_flags+=("--manifests-only") flags+=("--max-components=") @@ -5544,8 +5554,13 @@ _oc_adm_release_extract() flags+=("--git=") two_word_flags+=("--git") local_nonpersistent_flags+=("--git=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5645,6 +5660,9 @@ _oc_adm_release_info() flags+=("--dir=") two_word_flags+=("--dir") local_nonpersistent_flags+=("--dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image-for=") two_word_flags+=("--image-for") local_nonpersistent_flags+=("--image-for=") @@ -5652,6 +5670,8 @@ _oc_adm_release_info() local_nonpersistent_flags+=("--include-images") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5746,8 +5766,13 @@ _oc_adm_release_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -5757,6 +5782,9 @@ _oc_adm_release_mirror() two_word_flags+=("--registry-config") two_word_flags+=("-a") local_nonpersistent_flags+=("--registry-config=") + flags+=("--release-image-icsp-to-dir=") + two_word_flags+=("--release-image-icsp-to-dir") + local_nonpersistent_flags+=("--release-image-icsp-to-dir=") flags+=("--release-image-signature-to-dir=") two_word_flags+=("--release-image-signature-to-dir") local_nonpersistent_flags+=("--release-image-signature-to-dir=") @@ -5865,11 +5893,16 @@ _oc_adm_release_new() flags+=("--from-release=") two_word_flags+=("--from-release") local_nonpersistent_flags+=("--from-release=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--include=") two_word_flags+=("--include") local_nonpersistent_flags+=("--include=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--mapping-file=") two_word_flags+=("--mapping-file") local_nonpersistent_flags+=("--mapping-file=") @@ -13508,11 +13541,16 @@ _oc_image_append() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image=") two_word_flags+=("--image") local_nonpersistent_flags+=("--image=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") @@ -13601,8 +13639,13 @@ _oc_image_extract() flags+=("--filter-by-os=") two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--only-files") local_nonpersistent_flags+=("--only-files") flags+=("--path=") @@ -13681,8 +13724,13 @@ _oc_image_info() flags+=("--filter-by-os=") two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--output=") two_word_flags+=("--output") two_word_flags+=("-o") @@ -13770,10 +13818,15 @@ _oc_image_mirror() flags+=("--from-dir=") two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--keep-manifest-list") local_nonpersistent_flags+=("--keep-manifest-list") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry=") diff --git a/pkg/cli/admin/catalog/build.go b/pkg/cli/admin/catalog/build.go index d66a277c65..fbe95689e9 100644 --- a/pkg/cli/admin/catalog/build.go +++ b/pkg/cli/admin/catalog/build.go @@ -96,7 +96,7 @@ func (o *BuildImageOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, arg } imageFromRealeaseTags := func(img string) { - info, err := infoOpts.LoadReleaseInfo(img, false) + info, err := infoOpts.LoadReleaseInfo(img, false, false, "") if err != nil { klog.V(2).Infof("unable to load image from %s: %v", img, err) return diff --git a/pkg/cli/admin/catalog/mirror.go b/pkg/cli/admin/catalog/mirror.go index 54ea5764ca..00bfe74fce 100644 --- a/pkg/cli/admin/catalog/mirror.go +++ b/pkg/cli/admin/catalog/mirror.go @@ -115,7 +115,7 @@ func NewMirrorCatalog(f kcmdutil.Factory, streams genericclioptions.IOStreams) * Long: mirrorLong, Example: mirrorExample, Run: func(cmd *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(cmd, args)) + kcmdutil.CheckErr(o.Complete(f, cmd, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -137,7 +137,7 @@ func NewMirrorCatalog(f kcmdutil.Factory, streams genericclioptions.IOStreams) * return cmd } -func (o *MirrorCatalogOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *MirrorCatalogOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if len(args) < 2 { return fmt.Errorf("must specify source and dest") } @@ -291,7 +291,7 @@ func (o *MirrorCatalogOptions) Complete(cmd *cobra.Command, args []string) error } e.Paths = []string{o.DatabasePath} e.Confirm = true - if err := e.Complete(cmd, []string{o.SourceRef.String()}); err != nil { + if err := e.Complete(f, cmd, []string{o.SourceRef.String()}); err != nil { return "", err } if err := e.Validate(); err != nil { diff --git a/pkg/cli/admin/release/extract.go b/pkg/cli/admin/release/extract.go index bca93fa435..89c86ff92d 100644 --- a/pkg/cli/admin/release/extract.go +++ b/pkg/cli/admin/release/extract.go @@ -174,6 +174,12 @@ func (o *ExtractOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args [ } o.From = args[0] + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } + if !o.SecurityOptions.LookupClusterICSP && len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.TryAlternativeSources = true + } return nil } @@ -240,7 +246,16 @@ func (o *ExtractOptions) Run() error { opts.FileDir = o.FileDir switch { + // o.File > 0 when mirroring release case len(o.File) > 0: + // set ref if using alternative image sources + regContext, err := opts.SecurityOptions.Context() + if err != nil { + return err + } + newRef, _, _, err := opts.SecurityOptions.PreferredImageSource(ref.Ref, regContext) + ref.Ref = newRef + src = ref.String() if o.ImageMetadataCallback != nil { opts.ImageMetadataCallback = o.ImageMetadataCallback } @@ -422,7 +437,8 @@ func (o *ExtractOptions) extractGit(dir string) error { opts := NewInfoOptions(o.IOStreams) opts.SecurityOptions = o.SecurityOptions opts.FileDir = o.FileDir - release, err := opts.LoadReleaseInfo(o.From, false) + + release, err := opts.LoadReleaseInfo(o.From, false, false, "") if err != nil { return err } diff --git a/pkg/cli/admin/release/extract_tools.go b/pkg/cli/admin/release/extract_tools.go index 5e1121a6f5..2a453eebdf 100644 --- a/pkg/cli/admin/release/extract_tools.go +++ b/pkg/cli/admin/release/extract_tools.go @@ -297,16 +297,153 @@ func (o *ExtractOptions) extractCommand(command string) error { infoOptions := NewInfoOptions(o.IOStreams) infoOptions.SecurityOptions = o.SecurityOptions infoOptions.FileDir = o.FileDir - release, err := infoOptions.LoadReleaseInfo(o.From, false) - if err != nil { - return err + + var releaseFromRefErr error + targetRelease := targetReleaseInfo{} + if len(o.SecurityOptions.ICSPFile) > 0 { + o.SecurityOptions.AddImageSourcePoliciesFromFile() + releaseFromImageSources, err := infoOptions.LoadReleaseInfo(o.From, false, true, o.SecurityOptions.ICSPFile) + if err != nil { + return fmt.Errorf("could not load release %s from icsp file %s: %v", o.From, o.SecurityOptions.ICSPFile, err) + } + targetRelease, err = o.setReleaseLookup(releaseFromImageSources, targets, currentOS, willArchive) + if err != nil { + // if icsp-file set, then return error if lookup from icsp fails + return fmt.Errorf("failed lookup of release %s from icsp file %s: %v", o.From, o.SecurityOptions.ICSPFile, err) + } + } else { + // This will be tried first, if no ImageContentSourcePolicyFile passed + releaseFromRef, loadReleaseFromRefErr := infoOptions.LoadReleaseInfo(o.From, false, false, "") + if loadReleaseFromRefErr == nil { + targetRelease, releaseFromRefErr = o.setReleaseLookup(releaseFromRef, targets, currentOS, willArchive) + // This will be returned if further lookup fails + if releaseFromRefErr != nil { + // If there's an error, now look for other imageSources - icsp and/or the user-passed image registry/repo/name + klog.V(2).Infof("Failed lookup of release from its reference: %v", releaseFromRefErr) + // now try other sources + releaseFromImageSources, err := infoOptions.LoadReleaseInfo(o.From, false, true, "") + if err != nil { + return err + } + targetRelease, err = o.setReleaseLookup(releaseFromImageSources, targets, currentOS, willArchive) + if err != nil { + klog.V(2).Infof("Failed lookup of release: %v", err) + return releaseFromRefErr + } + } + } else { + klog.V(2).Infof("Failed to load release info from image reference: %v", loadReleaseFromRefErr) + // now try other sources + releaseFromImageSources, err := infoOptions.LoadReleaseInfo(o.From, false, true, "") + if err != nil { + return err + } + targetRelease, err = o.setReleaseLookup(releaseFromImageSources, targets, currentOS, willArchive) + if err != nil { + klog.V(2).Infof("Failed lookup of release: %v", err) + return loadReleaseFromRefErr + } + } + } + if targetRelease.willArchive { + buf := &bytes.Buffer{} + fmt.Fprintf(buf, heredoc.Doc(` + Client tools for OpenShift + -------------------------- + + These archives contain the client tooling for [OpenShift](https://docs.openshift.com). + + To verify the contents of this directory, use the 'gpg' and 'shasum' tools to + ensure the archives you have downloaded match those published from this location. + + The openshift-install binary has been preconfigured to install the following release: + + --- + + `)) + if err := describeReleaseInfo(buf, targetRelease.release, false, false, true, false); err != nil { + return err + } + filename := "release.txt" + if err := ioutil.WriteFile(filepath.Join(dir, filename), buf.Bytes(), 0644); err != nil { + return err + } + hash := hashFn() + hash.Write(buf.Bytes()) + targetRelease.hashByTargetName[filename] = hex.EncodeToString(hash.Sum(nil)) + } + + // write a checksum of the tar files to disk as sha256sum.txt.asc + if len(targetRelease.hashByTargetName) > 0 { + var keys []string + for k := range targetRelease.hashByTargetName { + keys = append(keys, k) + } + sort.Strings(keys) + var lines []string + for _, k := range keys { + hash := targetRelease.hashByTargetName[k] + lines = append(lines, fmt.Sprintf("%s %s", hash, filepath.Base(k))) + } + // ensure a trailing newline + if len(lines[len(lines)-1]) != 0 { + lines = append(lines, "") + } + // write the content manifest + data := []byte(strings.Join(lines, "\n")) + filename := "sha256sum.txt" + if err := ioutil.WriteFile(filepath.Join(dir, filename), data, 0644); err != nil { + return fmt.Errorf("unable to write checksum file: %v", err) + } + // sign the content manifest + if signer != nil { + buf := &bytes.Buffer{} + if err := openpgp.ArmoredDetachSign(buf, signer, bytes.NewBuffer(data), nil); err != nil { + return fmt.Errorf("unable to sign the sha256sum.txt file: %v", err) + } + if err := ioutil.WriteFile(filepath.Join(dir, filename+".asc"), buf.Bytes(), 0644); err != nil { + return fmt.Errorf("unable to write signed manifest: %v", err) + } + } + } + + // if we did not process some targets, report that to the user and error if necessary + if len(targetRelease.targetsByName) > 0 { + var missing []string + for _, target := range targetRelease.targetsByName { + missing = append(missing, target.Mapping.From) + } + sort.Strings(missing) + if len(missing) == 1 { + return fmt.Errorf("image did not contain %s", missing[0]) + } + return fmt.Errorf("unable to find multiple files: %s", strings.Join(missing, ", ")) } + + return nil +} + +type targetReleaseInfo struct { + release *ReleaseInfo + willArchive bool + hashByTargetName map[string]string + targetsByName map[string]extractTarget +} + +func (o *ExtractOptions) setReleaseLookup(release *ReleaseInfo, targets []extractTarget, currentOS string, toArchive bool) (targetReleaseInfo, error) { releaseName := release.PreferredName() refExact := release.ImageRef refExact.Ref.Tag = "" refExact.Ref.ID = release.Digest.String() exactReleaseImage := refExact.String() + tr := targetReleaseInfo{ + release: release, + willArchive: toArchive, + hashByTargetName: make(map[string]string), + targetsByName: make(map[string]extractTarget), + } + // resolve target image references to their pull specs missing := sets.NewString() var validTargets []extractTarget @@ -323,16 +460,16 @@ func (o *ExtractOptions) extractCommand(command string) error { klog.V(2).Infof("Will extract %s from %s", target.Mapping.From, spec) ref, err := imagereference.Parse(spec) if err != nil { - return err + return targetReleaseInfo{}, err } target.Mapping.Image = spec target.Mapping.ImageRef = imagesource.TypedImageReference{Ref: ref, Type: imagesource.DestinationRegistry} if target.AsArchive { - willArchive = true + tr.willArchive = true target.Mapping.Name = fmt.Sprintf(target.ArchiveFormat, releaseName) - target.Mapping.To = filepath.Join(dir, target.Mapping.Name) + target.Mapping.To = filepath.Join(o.Directory, target.Mapping.Name) } else { - target.Mapping.To = filepath.Join(dir, target.Command) + target.Mapping.To = filepath.Join(o.Directory, target.Command) target.Mapping.Name = fmt.Sprintf("%s-%s", target.OS, target.Command) } validTargets = append(validTargets, target) @@ -340,9 +477,9 @@ func (o *ExtractOptions) extractCommand(command string) error { if len(validTargets) == 0 { if len(missing) == 1 { - return fmt.Errorf("the image %q containing the desired command is not available", missing.List()[0]) + return targetReleaseInfo{}, fmt.Errorf("the image %q containing the desired command is not available", missing.List()[0]) } - return fmt.Errorf("some required images are missing: %s", strings.Join(missing.List(), ", ")) + return targetReleaseInfo{}, fmt.Errorf("some required images are missing: %s", strings.Join(missing.List(), ", ")) } if len(missing) > 0 { fmt.Fprintf(o.ErrOut, "warning: Some commands can not be extracted due to missing images: %s\n", strings.Join(missing.List(), ", ")) @@ -356,16 +493,13 @@ func (o *ExtractOptions) extractCommand(command string) error { // create the mapping lookup of the valid targets var extractLock sync.Mutex - targetsByName := make(map[string]extractTarget) for _, target := range validTargets { - targetsByName[target.Mapping.Name] = target + tr.targetsByName[target.Mapping.Name] = target opts.Mappings = append(opts.Mappings, target.Mapping) } - hashByTargetName := make(map[string]string) - // ensure to is a directory - if err := os.MkdirAll(dir, 0777); err != nil { - return err + if err := os.MkdirAll(o.Directory, 0777); err != nil { + return targetReleaseInfo{}, err } // as each layer is extracted, take the output binary and write it to disk @@ -374,7 +508,7 @@ func (o *ExtractOptions) extractCommand(command string) error { target, ok := func() (extractTarget, bool) { extractLock.Lock() defer extractLock.Unlock() - target, ok := targetsByName[layer.Mapping.Name] + target, ok := tr.targetsByName[layer.Mapping.Name] return target, ok }() if !ok { @@ -395,6 +529,7 @@ func (o *ExtractOptions) extractCommand(command string) error { w = bw var hash hash.Hash + var hashFn = sha256.New closeFn := func() error { return nil } if target.AsArchive { text := strings.Replace(target.Readme, `\u0060`, "`", -1) @@ -535,94 +670,18 @@ func (o *ExtractOptions) extractCommand(command string) error { func() { extractLock.Lock() defer extractLock.Unlock() - delete(targetsByName, layer.Mapping.Name) + delete(tr.targetsByName, layer.Mapping.Name) if hash != nil { - hashByTargetName[layer.Mapping.To] = hex.EncodeToString(hash.Sum(nil)) + tr.hashByTargetName[layer.Mapping.To] = hex.EncodeToString(hash.Sum(nil)) } }() return false, nil } if err := opts.Run(); err != nil { - return err - } - - if willArchive { - buf := &bytes.Buffer{} - fmt.Fprintf(buf, heredoc.Doc(` - Client tools for OpenShift - -------------------------- - - These archives contain the client tooling for [OpenShift](https://docs.openshift.com). - - To verify the contents of this directory, use the 'gpg' and 'shasum' tools to - ensure the archives you have downloaded match those published from this location. - - The openshift-install binary has been preconfigured to install the following release: - - --- - - `)) - if err := describeReleaseInfo(buf, release, false, false, true, false); err != nil { - return err - } - filename := "release.txt" - if err := ioutil.WriteFile(filepath.Join(dir, filename), buf.Bytes(), 0644); err != nil { - return err - } - hash := hashFn() - hash.Write(buf.Bytes()) - hashByTargetName[filename] = hex.EncodeToString(hash.Sum(nil)) + return targetReleaseInfo{}, err } - - // write a checksum of the tar files to disk as sha256sum.txt.asc - if len(hashByTargetName) > 0 { - var keys []string - for k := range hashByTargetName { - keys = append(keys, k) - } - sort.Strings(keys) - var lines []string - for _, k := range keys { - hash := hashByTargetName[k] - lines = append(lines, fmt.Sprintf("%s %s", hash, filepath.Base(k))) - } - // ensure a trailing newline - if len(lines[len(lines)-1]) != 0 { - lines = append(lines, "") - } - // write the content manifest - data := []byte(strings.Join(lines, "\n")) - filename := "sha256sum.txt" - if err := ioutil.WriteFile(filepath.Join(dir, filename), data, 0644); err != nil { - return fmt.Errorf("unable to write checksum file: %v", err) - } - // sign the content manifest - if signer != nil { - buf := &bytes.Buffer{} - if err := openpgp.ArmoredDetachSign(buf, signer, bytes.NewBuffer(data), nil); err != nil { - return fmt.Errorf("unable to sign the sha256sum.txt file: %v", err) - } - if err := ioutil.WriteFile(filepath.Join(dir, filename+".asc"), buf.Bytes(), 0644); err != nil { - return fmt.Errorf("unable to write signed manifest: %v", err) - } - } - } - - // if we did not process some targets, report that to the user and error if necessary - if len(targetsByName) > 0 { - var missing []string - for _, target := range targetsByName { - missing = append(missing, target.Mapping.From) - } - sort.Strings(missing) - if len(missing) == 1 { - return fmt.Errorf("image did not contain %s", missing[0]) - } - return fmt.Errorf("unable to find multiple files: %s", strings.Join(missing, ", ")) - } - - return nil + return tr, nil } const ( diff --git a/pkg/cli/admin/release/info.go b/pkg/cli/admin/release/info.go index ebe5b1fb6c..f8ef31cc1e 100644 --- a/pkg/cli/admin/release/info.go +++ b/pkg/cli/admin/release/info.go @@ -367,6 +367,12 @@ func (o *InfoOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []st o.From = o.Images[0] o.Images = o.Images[1:] } + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } + if !o.SecurityOptions.LookupClusterICSP && len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.TryAlternativeSources = true + } return nil } @@ -454,10 +460,10 @@ func (o *InfoOptions) Run() error { done := make(chan struct{}) go func() { defer close(done) - baseRelease, baseErr = o.LoadReleaseInfo(o.From, fetchImages) + baseRelease, baseErr = o.LoadReleaseInfo(o.From, fetchImages, false, "") }() - release, err := o.LoadReleaseInfo(o.Images[0], fetchImages) + release, err := o.LoadReleaseInfo(o.Images[0], fetchImages, false, "") if err != nil { return err } @@ -482,7 +488,7 @@ func (o *InfoOptions) Run() error { var exitErr error for _, image := range o.Images { - release, err := o.LoadReleaseInfo(image, fetchImages) + release, err := o.LoadReleaseInfo(image, fetchImages, false, "") if err != nil { exitErr = kcmdutil.ErrExit fmt.Fprintf(o.ErrOut, "error: %v\n", err) @@ -734,16 +740,32 @@ func (i *ReleaseInfo) Platform() string { return fmt.Sprintf("%s/%s", os, arch) } -func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool) (*ReleaseInfo, error) { +func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool, setImageSourcePrefix bool, icspFile string) (*ReleaseInfo, error) { + replacedRef := false + opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) + opts.SecurityOptions = o.SecurityOptions + opts.FileDir = o.FileDir ref, err := imagesource.ParseReference(image) if err != nil { return nil, err } + fromContext, err := opts.SecurityOptions.Context() + if err != nil { + return nil, err + } + + if setImageSourcePrefix { + newRef, _, _, err := opts.SecurityOptions.PreferredImageSource(ref.Ref, fromContext) + if err != nil { + return nil, err + } + replacedRef = true + image = newRef.String() + ref.Ref = newRef + } + verifier := imagemanifest.NewVerifier() - opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) - opts.SecurityOptions = o.SecurityOptions - opts.FileDir = o.FileDir release := &ReleaseInfo{ Image: image, @@ -783,6 +805,26 @@ func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool) (*Relea errs = append(errs, err) return true, nil } + if replacedRef { + sourceOpts := &imagesource.Options{FileDir: opts.FileDir, Insecure: opts.SecurityOptions.Insecure, RegistryContext: fromContext} + for _, tag := range is.Spec.Tags { + // If can't get digest ID, skip this tag, this happens when user has built a payload by + // replacing component images in the release with a new image + // imageSources is slice of 'registry/repo/name' determined from ICSP + tagRef, err := imagereference.Parse(tag.From.Name) + // if err != nil, skip this tag + if err == nil { + ref.Ref.ID = tagRef.ID + srcRef, err := imagesource.ParseReference(ref.String()) + if err != nil { + return true, err + } + if _, _, err = sourceOpts.RepositoryWithManifests(context.TODO(), srcRef); err == nil { + tag.From.Name = ref.String() + } + } + } + } release.References = is case "release-metadata": data, err := ioutil.ReadAll(r) @@ -835,7 +877,7 @@ func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool) (*Relea var lock sync.Mutex release.Images = make(map[string]*Image) r := &imageinfo.ImageRetriever{ - FileDir: opts.FileDir, + FileDir: o.FileDir, Image: make(map[string]imagesource.TypedImageReference), SecurityOptions: o.SecurityOptions, ParallelOptions: o.ParallelOptions, diff --git a/pkg/cli/admin/release/mirror.go b/pkg/cli/admin/release/mirror.go index bfa9c55b3d..21e4436e0c 100644 --- a/pkg/cli/admin/release/mirror.go +++ b/pkg/cli/admin/release/mirror.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "io/ioutil" "net/http" "os" @@ -50,8 +49,9 @@ import ( ) // configFilesBaseDir is created under '--to-dir', when specified, to contain release image -// signature files. It is not used when '--release-image-signature-to-dir` is specified -// which takes precedence over '--to-dir'. +// signature and icsp files. It is not used for signature file when '--release-image-signature-to-dir' is specified. +// It is not used for icsp file when '--release-image-icsp-to-dir' is specified. +// The signature-to and icsp-to flags take precedence over '--to-dir'. const configFilesBaseDir = "config" // maxDigestHashLen is used to truncate digest hash portion before using as part of @@ -61,6 +61,9 @@ const maxDigestHashLen = 16 // signatureFileNameFmt defines format of the release image signature file name const signatureFileNameFmt = "signature-%s-%s.yaml" +// icspFileNameFmt defines format of the release image ImageContentSourcePolicy file name +const icspFileNameFmt = "icsp-%s-%s.yaml" + // NewMirrorOptions creates the options for mirroring a release. func NewMirrorOptions(streams genericclioptions.IOStreams) *MirrorOptions { return &MirrorOptions{ @@ -106,30 +109,36 @@ func NewMirror(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C that can be used to upload the release to another registry. You may use --apply-release-image-signature, --release-image-signature-to-dir, or both - to control the handling of the signature ConfigMap. Option - --apply-release-image-signature will apply the ConfigMap directly to a connected - cluster while --release-image-signature-to-dir specifies an export target directory. If + to control the handling of the signature ConfigMap file. Option + --apply-release-image-signature will apply the release signature ConfigMap directly to a connected + cluster while --release-image-signature-to-dir specifies an export target directory. If --release-image-signature-to-dir is not specified but --to-dir is, --release-image-signature-to-dir defaults to a 'config' subdirectory of --to-dir. The --overwrite option only applies when --apply-release-image-signature is specified and indicates to update an exisiting ConfigMap if one is found. A ConfigMap written to a directory will always replace onethat already exists. + + You may use --release-image-icsp-to-dir to specifiy the export target directory of the + ImageContentSourcePolicy file. The default is 'config'. If not specified but --to-dir is, + --release-image-icsp-to-dir defaults to a 'config' subdirectory of --to-dir. `), Example: templates.Examples(` - # Perform a dry run showing what would be mirrored, including the mirror objects + # Perform a dry run showing what would be mirrored, including the mirror objects, and control where signature and ICSP files are written oc adm release mirror 4.3.0 --to myregistry.local/openshift/release \ - --release-image-signature-to-dir /tmp/releases --dry-run + --release-image-signature-to-dir /tmp/releases \ + --release-image-icsp-to-dir /tmp/icsps --dry-run - # Mirror a release into the current directory + # Mirror a release into the current directory and control where signature and ICSP files are written oc adm release mirror 4.3.0 --to file://openshift/release \ - --release-image-signature-to-dir /tmp/releases + --release-image-signature-to-dir /tmp/releases \ + --release-image-icsp-to-dir /tmp/icsps # Mirror a release to another directory in the default location oc adm release mirror 4.3.0 --to-dir /tmp/releases # Upload a release from the current directory to another server oc adm release mirror --from file://openshift/release --to myregistry.com/openshift/release \ - --release-image-signature-to-dir /tmp/releases + --release-image-config-dir /tmp/releases # Mirror the 4.3.0 release to repository registry.example.com and apply signatures to connected cluster oc adm release mirror --from=quay.io/openshift-release-dev/ocp-release:4.3.0-x86_64 \ @@ -150,10 +159,11 @@ func NewMirror(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C flags.StringVar(&o.ToImageStream, "to-image-stream", o.ToImageStream, "An image stream to tag images into.") flags.StringVar(&o.FromDir, "from-dir", o.FromDir, "A directory to import images from.") flags.StringVar(&o.ToDir, "to-dir", o.ToDir, "A directory to export images to.") + flags.StringVar(&o.ReleaseImageICSPToDir, "release-image-icsp-to-dir", o.ReleaseImageICSPToDir, "Path to write ImageContentSourcePolicy file. If not set, defaults to 'config'.") + flags.StringVar(&o.ReleaseImageSignatureToDir, "release-image-signature-to-dir", o.ReleaseImageSignatureToDir, "Path to write release image signature ConfigMap files. If not set, defaults to 'config'.") flags.BoolVar(&o.ToMirror, "to-mirror", o.ToMirror, "Output the mirror mappings instead of mirroring.") flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Display information about the mirror without actually executing it.") flags.BoolVar(&o.ApplyReleaseImageSignature, "apply-release-image-signature", o.ApplyReleaseImageSignature, "Apply release image signature to connected cluster.") - flags.StringVar(&o.ReleaseImageSignatureToDir, "release-image-signature-to-dir", o.ReleaseImageSignatureToDir, "A directory to export release image signature to.") flags.BoolVar(&o.SkipRelease, "skip-release-image", o.SkipRelease, "Do not push the release image.") flags.StringVar(&o.ToRelease, "to-release-image", o.ToRelease, "Specify an alternate locations for the release image instead as tag 'release' in --to.") @@ -182,10 +192,10 @@ type MirrorOptions struct { ApplyReleaseImageSignature bool ReleaseImageSignatureToDir string + ReleaseImageICSPToDir string Overwrite bool - DryRun bool - PrintImageContentInstructions bool + DryRun bool ImageClientFn func() (imageclient.Interface, string, error) CoreV1ClientFn func() (corev1client.ConfigMapInterface, error) @@ -215,6 +225,13 @@ func (o *MirrorOptions) Complete(cmd *cobra.Command, f kcmdutil.Factory, args [] } o.From = args[0] + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } + if !o.SecurityOptions.LookupClusterICSP && len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.TryAlternativeSources = true + } + o.ImageClientFn = func() (imageclient.Interface, string, error) { cfg, err := f.ToRESTConfig() if err != nil { @@ -242,7 +259,6 @@ func (o *MirrorOptions) Complete(cmd *cobra.Command, f kcmdutil.Factory, args [] client := coreClient.ConfigMaps(configmap.NamespaceLabelConfigMap) return client, nil } - o.PrintImageContentInstructions = true return nil } @@ -280,6 +296,10 @@ func (o *MirrorOptions) Validate() error { o.ReleaseImageSignatureToDir = filepath.Join(o.ToDir, configFilesBaseDir) } + if len(o.ReleaseImageICSPToDir) == 0 && len(o.ToDir) > 0 { + o.ReleaseImageICSPToDir = filepath.Join(o.ToDir, configFilesBaseDir) + } + if o.Overwrite && !o.ApplyReleaseImageSignature { return fmt.Errorf("--overwite is only valid when --apply-release-image-signature is specified") } @@ -312,6 +332,19 @@ func createSignatureFileName(digest string) (string, error) { return fmt.Sprintf(signatureFileNameFmt, algo, hash), nil } +func createICSPFileName(digest string) (string, error) { + parts := strings.SplitN(digest, ":", 3) + if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 { + return "", fmt.Errorf("the provided digest, %s, must be of the form ALGO:HASH", digest) + } + algo, hash := parts[0], parts[1] + + if len(hash) > maxDigestHashLen { + hash = hash[:maxDigestHashLen] + } + return fmt.Sprintf(icspFileNameFmt, algo, hash), nil +} + // handleSignatures implements the image release signature configmap specific logic. // Signature configmaps may be written to a directory or applied to a cluster. func (o *MirrorOptions) handleSignatures(context context.Context, signaturesByDigest map[string][][]byte) error { @@ -520,10 +553,11 @@ func (o *MirrorOptions) Run() error { if err := imageVerifier.Verify(ctx, releaseDigest); err != nil { fmt.Fprintf(o.ErrOut, "warning: An image was retrieved that failed verification: %v\n", err) } + var srcRef imagesource.TypedImageReference var mappings []mirror.Mapping if len(o.From) > 0 { src := o.From - srcRef, err := imagesource.ParseReference(src) + srcRef, err = imagesource.ParseReference(src) if err != nil { return fmt.Errorf("invalid --from: %v", err) } @@ -531,16 +565,11 @@ func (o *MirrorOptions) Run() error { // if the source ref is a file type, provide a function that checks the local file store for a given manifest // before continuing, to allow mirroring an entire release to disk in a single file://REPO. if srcRef.Type == imagesource.DestinationFile { - if repo, err := (&imagesource.Options{FileDir: o.FromDir}).Repository(context.TODO(), srcRef); err == nil { + if _, manifests, err := (&imagesource.Options{FileDir: o.FromDir}).RepositoryWithManifests(context.TODO(), srcRef); err == nil { sourceFn = func(ref imagesource.TypedImageReference) imagesource.TypedImageReference { if ref.Type == imagesource.DestinationFile || len(ref.Ref.ID) == 0 { return ref } - manifests, err := repo.Manifests(context.TODO()) - if err != nil { - klog.V(2).Infof("Unable to get local manifest service: %v", err) - return ref - } ok, err := manifests.Exists(context.TODO(), digest.Digest(ref.Ref.ID)) if err != nil { klog.V(2).Infof("Unable to get check for local manifest: %v", err) @@ -786,6 +815,13 @@ func (o *MirrorOptions) Run() error { fmt.Fprintf(o.Out, "Mirrored to: %s\n", t) } } + if len(o.ReleaseImageICSPToDir) == 0 { + o.ReleaseImageICSPToDir = configFilesBaseDir + } + + if err := o.printImageContentInstructions(repositories, toList, releaseDigest); err != nil { + return fmt.Errorf("Error creating mirror usage instructions: %v", err) + } } if toDisk { if len(o.ToDir) > 0 { @@ -793,12 +829,6 @@ func (o *MirrorOptions) Run() error { } else { fmt.Fprintf(o.Out, "\nTo upload local images to a registry, run:\n\n oc image mirror 'file://%s*' REGISTRY/REPOSITORY\n\n", to) } - } else if len(toList) > 0 { - if o.PrintImageContentInstructions { - if err := printImageContentInstructions(o.Out, o.From, toList, o.ReleaseImageSignatureToDir, repositories); err != nil { - return fmt.Errorf("Error creating mirror usage instructions: %v", err) - } - } } if o.ApplyReleaseImageSignature || len(o.ReleaseImageSignatureToDir) > 0 { signatures := imageVerifier.Signatures() @@ -824,15 +854,17 @@ func (o *MirrorOptions) Run() error { // printImageContentInstructions provides examples to the user for using the new repository mirror // https://github.com/openshift/installer/blob/master/docs/dev/alternative_release_image_sources.md -func printImageContentInstructions(out io.Writer, from string, toList []string, signatureToDir string, repositories map[string]struct{}) error { +func (o *MirrorOptions) printImageContentInstructions(repositories map[string]struct{}, toList []string, digest string) error { type installConfigSubsection struct { ImageContentSources []operatorv1alpha1.RepositoryDigestMirrors `json:"imageContentSources"` } var sources []operatorv1alpha1.RepositoryDigestMirrors + var mirrorRef imagesource.TypedImageReference + var err error for _, to := range toList { - mirrorRef, err := imagesource.ParseReference(to) + mirrorRef, err = imagesource.ParseReference(to) if err != nil { return fmt.Errorf("Unable to parse image reference '%s': %v", to, err) } @@ -841,10 +873,10 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, } mirrorRepo := mirrorRef.Ref.AsRepository().String() - if len(from) != 0 { - sourceRef, err := imagesource.ParseReference(from) + if len(o.From) != 0 { + sourceRef, err := imagesource.ParseReference(o.From) if err != nil { - return fmt.Errorf("Unable to parse image reference '%s': %v", from, err) + return fmt.Errorf("Unable to parse image reference '%s': %v", o.From, err) } if sourceRef.Type != imagesource.DestinationRegistry { return nil @@ -852,7 +884,6 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, sourceRepo := sourceRef.Ref.AsRepository().String() repositories[sourceRepo] = struct{}{} } - if len(repositories) == 0 { return nil } @@ -875,16 +906,20 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, if err != nil { return fmt.Errorf("Unable to marshal install-config.yaml example yaml: %v", err) } - fmt.Fprintf(out, "\nTo use the new mirrored repository to install, add the following section to the install-config.yaml:\n\n") - fmt.Fprintf(out, string(installConfigExample)) + fmt.Fprintf(o.Out, "\nTo use the new mirrored repository to install, add the following section to the install-config.yaml:\n\n") + fmt.Fprintf(o.Out, string(installConfigExample)) + + // Create and display ImageContentSourcePolicy + // last mirrorRef will be used to name icsp + mirrorRepoStripped := strings.FieldsFunc(mirrorRef.Ref.Name, func(r rune) bool { return strings.ContainsRune(" .:/", r) }) + icspName := strings.Join(mirrorRepoStripped[:], "-") - // Create and display ImageContentSourcePolicy example icsp := operatorv1alpha1.ImageContentSourcePolicy{ TypeMeta: metav1.TypeMeta{ APIVersion: operatorv1alpha1.GroupVersion.String(), Kind: "ImageContentSourcePolicy"}, ObjectMeta: metav1.ObjectMeta{ - Name: "example", + Name: icspName, }, Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ RepositoryDigestMirrors: sources, @@ -899,16 +934,30 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, } delete(unstructuredObj.Object["metadata"].(map[string]interface{}), "creationTimestamp") - icspExample, err := yaml.Marshal(unstructuredObj.Object) + icspDataBytes, err := yaml.Marshal(unstructuredObj.Object) if err != nil { - return fmt.Errorf("Unable to marshal ImageContentSourcePolicy example yaml: %v", err) + return fmt.Errorf("Unable to marshal ImageContentSourcePolicy yaml: %v", err) } - fmt.Fprintf(out, "\n\nTo use the new mirrored repository for upgrades, use the following to create an ImageContentSourcePolicy:\n\n") - fmt.Fprintf(out, string(icspExample)) - - if len(signatureToDir) != 0 { - fmt.Fprintf(out, "\n\nTo apply signature configmaps use 'oc apply' on files found in %s\n\n", signatureToDir) + icspFileName, err := createICSPFileName(digest) + if err != nil { + return fmt.Errorf("creating filename: %v", err) } + icspFullName := filepath.Join(o.ReleaseImageICSPToDir, icspFileName) + if o.DryRun { + fmt.Fprintf(o.Out, "\ninfo: Write ImageContentSourcePolicy file %s\n", icspFullName) + } else { + if err := os.MkdirAll(filepath.Dir(icspFullName), 0750); err != nil { + return err + } + if err := ioutil.WriteFile(icspFullName, icspDataBytes, 0640); err != nil { + return err + } + fmt.Fprintf(o.Out, "\nImageContentSourcePolicy file %s created\n", icspFullName) + } + fmt.Fprintf(o.Out, "\n\nTo use the new mirrored repository for upgrades, use the following to create an ImageContentSourcePolicy:\n\n") + fmt.Fprintf(o.Out, string(icspDataBytes)) + + fmt.Fprintf(o.Out, "\n\nTo apply ImageContentSourcePolicy, use 'oc apply' on file %s\n\n", icspFullName) return nil } diff --git a/pkg/cli/admin/release/new.go b/pkg/cli/admin/release/new.go index 7bd37cd880..0f1455763e 100644 --- a/pkg/cli/admin/release/new.go +++ b/pkg/cli/admin/release/new.go @@ -246,6 +246,12 @@ func (o *NewOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []str o.Namespace = namespace } } + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } + if !o.SecurityOptions.LookupClusterICSP && len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.TryAlternativeSources = true + } return nil } diff --git a/pkg/cli/admin/verifyimagesignature/manifest.go b/pkg/cli/admin/verifyimagesignature/manifest.go index e0b0c16504..a3d5816566 100644 --- a/pkg/cli/admin/verifyimagesignature/manifest.go +++ b/pkg/cli/admin/verifyimagesignature/manifest.go @@ -31,12 +31,10 @@ func getImageManifestByIDFromRegistry(registry *url.URL, repositoryName, imageID if err != nil { return nil, err } - - manifests, err := repo.Manifests(ctx, nil) + manifests, err := repo.Manifests(ctx) if err != nil { return nil, err } - manifest, err := manifests.Get(ctx, godigest.Digest(imageID)) if err != nil { return nil, err diff --git a/pkg/cli/image/append/append.go b/pkg/cli/image/append/append.go index 1916f3d0b9..5ec6c52717 100644 --- a/pkg/cli/image/append/append.go +++ b/pkg/cli/image/append/append.go @@ -107,7 +107,7 @@ func NewAppendImageOptions(streams genericclioptions.IOStreams) *AppendImageOpti } // New creates a new command -func NewCmdAppendImage(streams genericclioptions.IOStreams) *cobra.Command { +func NewCmdAppendImage(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewAppendImageOptions(streams) cmd := &cobra.Command{ @@ -116,7 +116,7 @@ func NewCmdAppendImage(streams genericclioptions.IOStreams) *cobra.Command { Long: desc, Example: example, Run: func(c *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(c, args)) + kcmdutil.CheckErr(o.Complete(f, c, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -145,11 +145,13 @@ func NewCmdAppendImage(streams genericclioptions.IOStreams) *cobra.Command { return cmd } -func (o *AppendImageOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *AppendImageOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { return err } - + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } for _, arg := range args { if arg == "-" { if o.LayerStream != nil { @@ -210,50 +212,38 @@ func (o *AppendImageOptions) Run() error { } ctx := context.Background() + if len(o.FromFileDir) > 0 { + o.SecurityOptions.FileDir = o.FromFileDir + } + fromContext, err := o.SecurityOptions.Context() if err != nil { return err } - fromOptions := &imagesource.Options{ - FileDir: o.FileDir, - Insecure: o.SecurityOptions.Insecure, - RegistryContext: fromContext, - } - if len(o.FromFileDir) > 0 { - fromOptions.FileDir = o.FromFileDir - } toContext := fromContext.Copy().WithActions("pull", "push") - toOptions := &imagesource.Options{ - FileDir: o.FileDir, - Insecure: o.SecurityOptions.Insecure, - RegistryContext: toContext, - } - toRepo, err := toOptions.Repository(ctx, to) + newImageRef, toRepo, toManifests, err := o.SecurityOptions.PreferredImageSource(to.Ref, fromContext) if err != nil { return err } - toManifests, err := toRepo.Manifests(ctx) - if err != nil { - return err - } - + to.Ref = newImageRef var ( base *dockerv1client.DockerImageConfig baseDigest digest.Digest baseContentDigest digest.Digest layers []distribution.Descriptor fromRepo distribution.Repository + srcManifest distribution.Manifest + manifestLocation imagemanifest.ManifestLocation ) if from != nil { - repo, err := fromOptions.Repository(ctx, *from) + newFromRef, repo, _, err := o.SecurityOptions.PreferredImageSource(from.Ref, toContext) if err != nil { return err } - fromRepo = repo - - srcManifest, manifestLocation, err := imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) + from.Ref = newFromRef + srcManifest, manifestLocation, err = imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) if err != nil { return fmt.Errorf("unable to read image %s: %v", from, err) } @@ -261,7 +251,6 @@ func (o *AppendImageOptions) Run() error { if err != nil { return fmt.Errorf("unable to parse image %s: %v", from, err) } - contentDigest, err := registryclient.ContentDigestForManifest(srcManifest, manifestLocation.Manifest.Algorithm()) if err != nil { return err @@ -269,7 +258,7 @@ func (o *AppendImageOptions) Run() error { baseDigest = manifestLocation.Manifest baseContentDigest = contentDigest - + fromRepo = repo } else { base = add.NewEmptyConfig() layers = nil diff --git a/pkg/cli/image/extract/extract.go b/pkg/cli/image/extract/extract.go index 4f9d36c8d2..8632fcf464 100644 --- a/pkg/cli/image/extract/extract.go +++ b/pkg/cli/image/extract/extract.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/openshift/library-go/pkg/image/dockerv1client" + imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" "github.com/openshift/oc/pkg/cli/image/archive" "github.com/openshift/oc/pkg/cli/image/imagesource" @@ -135,7 +136,7 @@ func NewExtractOptions(streams genericclioptions.IOStreams) *ExtractOptions { } // New creates a new command -func NewExtract(streams genericclioptions.IOStreams) *cobra.Command { +func NewExtract(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewExtractOptions(streams) cmd := &cobra.Command{ @@ -144,7 +145,7 @@ func NewExtract(streams genericclioptions.IOStreams) *cobra.Command { Long: desc, Example: example, Run: func(c *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(c, args)) + kcmdutil.CheckErr(o.Complete(f, c, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -286,11 +287,10 @@ func parseMappings(images, paths, files []string, requireEmpty bool) ([]Mapping, return mappings, nil } -func (o *ExtractOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *ExtractOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { return err } - if len(args) == 0 { return fmt.Errorf("you must specify at least one image to extract as an argument") } @@ -304,6 +304,11 @@ func (o *ExtractOptions) Complete(cmd *cobra.Command, args []string) error { if err != nil { return err } + + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } + return nil } @@ -316,29 +321,26 @@ func (o *ExtractOptions) Validate() error { func (o *ExtractOptions) Run() error { ctx := context.Background() + var err error fromContext, err := o.SecurityOptions.Context() if err != nil { return err } - fromOptions := &imagesource.Options{ - FileDir: o.FileDir, - Insecure: o.SecurityOptions.Insecure, - RegistryContext: fromContext, - } - stopCh := make(chan struct{}) defer close(stopCh) q := workqueue.New(o.ParallelOptions.MaxPerRegistry, stopCh) return q.Try(func(q workqueue.Try) { + var repo distribution.Repository for i := range o.Mappings { mapping := o.Mappings[i] from := mapping.ImageRef q.Try(func() error { - repo, err := fromOptions.Repository(ctx, from) + var newImageRef imagereference.DockerImageReference + newImageRef, repo, _, err = o.SecurityOptions.PreferredImageSource(from.Ref, fromContext) if err != nil { - return fmt.Errorf("unable to connect to image repository %s: %v", from.String(), err) + return err } - + from.Ref = newImageRef srcManifest, location, err := imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) if err != nil { if imagemanifest.IsImageForbidden(err) { diff --git a/pkg/cli/image/image.go b/pkg/cli/image/image.go index 48d9a8351b..f6d860c176 100644 --- a/pkg/cli/image/image.go +++ b/pkg/cli/image/image.go @@ -35,16 +35,16 @@ func NewCmdImage(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra { Message: "View or copy images:", Commands: []*cobra.Command{ - info.NewInfo(streams), - mirror.NewCmdMirrorImage(streams), + info.NewInfo(f, streams), + mirror.NewCmdMirrorImage(f, streams), }, }, { Message: "Advanced commands:", Commands: []*cobra.Command{ serve.NewServe(streams), - append.NewCmdAppendImage(streams), - extract.NewExtract(streams), + append.NewCmdAppendImage(f, streams), + extract.NewExtract(f, streams), }, }, } diff --git a/pkg/cli/image/imagesource/file.go b/pkg/cli/image/imagesource/file.go index 9d2fb77b50..c37df7b3e3 100644 --- a/pkg/cli/image/imagesource/file.go +++ b/pkg/cli/image/imagesource/file.go @@ -26,16 +26,16 @@ type fileDriver struct { BaseDir string } -func (d *fileDriver) Repository(ctx context.Context, server *url.URL, repoName string, insecure bool) (distribution.Repository, error) { +func (d *fileDriver) Repository(ctx context.Context, server *url.URL, repoName string, insecure bool) (distribution.Repository, distribution.ManifestService, error) { klog.V(3).Infof("Repository %s %s", server, repoName) ref, err := reference.Parse(repoName) if err != nil { - return nil, err + return nil, nil, err } named, ok := ref.(reference.Named) if !ok { - return nil, fmt.Errorf("%s is not a valid repository name", repoName) + return nil, nil, fmt.Errorf("%s is not a valid repository name", repoName) } repo := &fileRepository{ @@ -43,7 +43,12 @@ func (d *fileDriver) Repository(ctx context.Context, server *url.URL, repoName s repoPath: repoPathForName(repoName), basePath: d.BaseDir, } - return repo, nil + + manifests, err := repo.Manifests(ctx) + if err != nil { + return nil, nil, fmt.Errorf("unable to get local manifest service: %v", err) + } + return repo, manifests, nil } func repoPathForName(repoName string) string { diff --git a/pkg/cli/image/imagesource/options.go b/pkg/cli/image/imagesource/options.go index 4662b0e680..caa09d7ed9 100644 --- a/pkg/cli/image/imagesource/options.go +++ b/pkg/cli/image/imagesource/options.go @@ -17,11 +17,20 @@ type Options struct { RegistryContext *registryclient.Context } -// Repository retrieves the appropriate repository implementation for the given typed reference. -func (o *Options) Repository(ctx context.Context, ref TypedImageReference) (distribution.Repository, error) { +// Repository retrieves the appropriate repository implementation and ManifestService for the given typed reference. +func (o *Options) RepositoryWithManifests(ctx context.Context, ref TypedImageReference) (distribution.Repository, distribution.ManifestService, error) { switch ref.Type { case DestinationRegistry: - return o.RegistryContext.Repository(ctx, ref.Ref.DockerClientDefaults().RegistryURL(), ref.Ref.RepositoryName(), o.Insecure) + repo, err := o.RegistryContext.Repository(ctx, ref.Ref.DockerClientDefaults().RegistryURL(), ref.Ref.RepositoryName(), o.Insecure) + if err != nil { + return nil, nil, err + } + manifests, err := repo.Manifests(ctx) + if err != nil { + return nil, nil, fmt.Errorf("unable to get local manifest service: %v", err) + } + return repo, manifests, nil + case DestinationFile: driver := &fileDriver{ BaseDir: o.FileDir, @@ -35,7 +44,7 @@ func (o *Options) Repository(ctx context.Context, ref TypedImageReference) (dist url := ref.Ref.DockerClientDefaults().RegistryURL() return driver.Repository(ctx, url, ref.Ref.RepositoryName(), o.Insecure) default: - return nil, fmt.Errorf("unrecognized image reference type %s", ref.Type) + return nil, nil, fmt.Errorf("unrecognized image reference type %s", ref.Type) } } @@ -48,7 +57,7 @@ func (o *Options) ExpandWildcard(ref TypedImageReference) ([]TypedImageReference } // lookup tags that match the search - repo, err := o.Repository(context.Background(), ref) + repo, _, err := o.RepositoryWithManifests(context.Background(), ref) if err != nil { return nil, err } diff --git a/pkg/cli/image/imagesource/s3.go b/pkg/cli/image/imagesource/s3.go index eaee32512b..a269b94e78 100644 --- a/pkg/cli/image/imagesource/s3.go +++ b/pkg/cli/image/imagesource/s3.go @@ -106,23 +106,23 @@ func (d *s3Driver) newObject(server *url.URL, region string, insecure bool, secu return s3obj, nil } -func (d *s3Driver) Repository(ctx context.Context, server *url.URL, repoName string, insecure bool) (distribution.Repository, error) { +func (d *s3Driver) Repository(ctx context.Context, server *url.URL, repoName string, insecure bool) (distribution.Repository, distribution.ManifestService, error) { parts := strings.SplitN(repoName, "/", 3) if len(parts) < 3 { - return nil, fmt.Errorf("you must pass a three segment repository name for s3 uploads, where the first segment is the region and the second segment is the bucket") + return nil, nil, fmt.Errorf("you must pass a three segment repository name for s3 uploads, where the first segment is the region and the second segment is the bucket") } s3obj, err := d.newObject(server, parts[0], insecure, &url.URL{Scheme: server.Scheme, Host: server.Host, Path: "/" + repoName}) if err != nil { - return nil, err + return nil, nil, err } ref, err := reference.Parse(parts[2]) if err != nil { - return nil, err + return nil, nil, err } named, ok := ref.(reference.Named) if !ok { - return nil, fmt.Errorf("%s is not a valid repository name", parts[2]) + return nil, nil, fmt.Errorf("%s is not a valid repository name", parts[2]) } repo := &s3Repository{ @@ -132,7 +132,11 @@ func (d *s3Driver) Repository(ctx context.Context, server *url.URL, repoName str repoName: named, copyFrom: d.CopyFrom, } - return repo, nil + manifests, err := repo.Manifests(context.TODO()) + if err != nil { + return nil, nil, fmt.Errorf("unable to get local manifest service: %v", err) + } + return repo, manifests, nil } type s3Repository struct { diff --git a/pkg/cli/image/info/info.go b/pkg/cli/image/info/info.go index be662ad6bb..670e7878e0 100644 --- a/pkg/cli/image/info/info.go +++ b/pkg/cli/image/info/info.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/openshift/library-go/pkg/image/dockerv1client" + imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" "github.com/openshift/oc/pkg/cli/image/imagesource" imagemanifest "github.com/openshift/oc/pkg/cli/image/manifest" @@ -37,7 +38,7 @@ func NewInfoOptions(streams genericclioptions.IOStreams) *InfoOptions { } } -func NewInfo(streams genericclioptions.IOStreams) *cobra.Command { +func NewInfo(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewInfoOptions(streams) cmd := &cobra.Command{ Use: "info IMAGE [...]", @@ -67,7 +68,7 @@ func NewInfo(streams genericclioptions.IOStreams) *cobra.Command { `), Run: func(cmd *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(cmd, args)) + kcmdutil.CheckErr(o.Complete(f, cmd, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -93,11 +94,14 @@ type InfoOptions struct { Output string } -func (o *InfoOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *InfoOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if len(args) < 1 { return fmt.Errorf("info expects at least one argument, an image pull spec") } o.Images = args + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } return nil } @@ -109,16 +113,15 @@ func (o *InfoOptions) Run() error { if len(o.Images) == 0 { return fmt.Errorf("must specify one or more images as arguments") } - - // cache the context - context, err := o.SecurityOptions.Context() + regContext, err := o.SecurityOptions.Context() if err != nil { return err } + opts := &imagesource.Options{ FileDir: o.FileDir, Insecure: o.SecurityOptions.Insecure, - RegistryContext: context, + RegistryContext: regContext, } hadError := false @@ -368,11 +371,6 @@ func (o *ImageRetriever) Run() error { if err != nil { return err } - fromOptions := &imagesource.Options{ - FileDir: o.FileDir, - Insecure: o.SecurityOptions.Insecure, - RegistryContext: fromContext, - } callbackFn := o.ImageMetadataCallback if callbackFn == nil { @@ -388,12 +386,14 @@ func (o *ImageRetriever) Run() error { name := key from := o.Image[key] q.Try(func() error { - repo, err := fromOptions.Repository(ctx, from) + var newImageRef imagereference.DockerImageReference + newImageRef, repo, manifests, err := o.SecurityOptions.PreferredImageSource(from.Ref, fromContext) if err != nil { return callbackFn(name, nil, fmt.Errorf("unable to connect to image repository %s: %v", from, err)) } + from.Ref = newImageRef - allManifests, manifestList, listDigest, err := imagemanifest.AllManifests(ctx, from.Ref, repo) + allManifests, manifestList, listDigest, err := imagemanifest.AllManifests(ctx, from.Ref, manifests, repo) if err != nil { if imagemanifest.IsImageForbidden(err) { msg := fmt.Sprintf("image %q does not exist or you don't have permission to access the repository", from) diff --git a/pkg/cli/image/manifest/manifest.go b/pkg/cli/image/manifest/manifest.go index 1f6106a7a6..8d0ebf862d 100644 --- a/pkg/cli/image/manifest/manifest.go +++ b/pkg/cli/image/manifest/manifest.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "net/http" "regexp" "runtime" "sync" @@ -21,13 +20,11 @@ import ( "github.com/docker/libtrust" "github.com/opencontainers/go-digest" - "k8s.io/client-go/rest" "k8s.io/klog/v2" "github.com/openshift/library-go/pkg/image/dockerv1client" imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" - "github.com/openshift/oc/pkg/cli/image/manifest/dockercredentials" "github.com/openshift/oc/pkg/helpers/image/dockerlayer/add" ) @@ -39,37 +36,6 @@ func (o *ParallelOptions) Bind(flags *pflag.FlagSet) { flags.IntVar(&o.MaxPerRegistry, "max-per-registry", o.MaxPerRegistry, "Number of concurrent requests allowed per registry.") } -type SecurityOptions struct { - RegistryConfig string - Insecure bool - SkipVerification bool - - CachedContext *registryclient.Context -} - -func (o *SecurityOptions) Bind(flags *pflag.FlagSet) { - flags.StringVarP(&o.RegistryConfig, "registry-config", "a", o.RegistryConfig, "Path to your registry credentials (defaults to ~/.docker/config.json)") - flags.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP") - flags.BoolVar(&o.SkipVerification, "skip-verification", o.SkipVerification, "Skip verifying the integrity of the retrieved content. This is not recommended, but may be necessary when importing images from older image registries. Only bypass verification if the registry is known to be trustworthy.") -} - -// ReferentialHTTPClient returns an http.Client that is appropriate for accessing -// blobs referenced outside of the registry (due to the present of the URLs attribute -// in the manifest reference for a layer). -func (o *SecurityOptions) ReferentialHTTPClient() (*http.Client, error) { - ctx, err := o.Context() - if err != nil { - return nil, err - } - client := &http.Client{} - if o.Insecure { - client.Transport = ctx.InsecureTransport - } else { - client.Transport = ctx.Transport - } - return client, nil -} - type Verifier interface { Verify(dgst, contentDgst digest.Digest) Verified() bool @@ -99,39 +65,6 @@ func (v *verifier) Verified() bool { return !v.hadError } -func (o *SecurityOptions) Context() (*registryclient.Context, error) { - if o.CachedContext != nil { - return o.CachedContext, nil - } - context, err := o.NewContext() - if err == nil { - o.CachedContext = context - o.CachedContext.Retries = 3 - } - return context, err -} - -func (o *SecurityOptions) NewContext() (*registryclient.Context, error) { - rt, err := rest.TransportFor(&rest.Config{}) - if err != nil { - return nil, err - } - insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}}) - if err != nil { - return nil, err - } - creds := dockercredentials.NewLocal() - if len(o.RegistryConfig) > 0 { - creds, err = dockercredentials.NewFromFile(o.RegistryConfig) - if err != nil { - return nil, fmt.Errorf("unable to load --registry-config: %v", err) - } - } - context := registryclient.NewContext(rt, insecureRT).WithCredentials(creds) - context.DisableDigestVerification = o.SkipVerification - return context, nil -} - // FilterOptions assist in filtering out unneeded manifests from ManifestList objects. type FilterOptions struct { FilterByOS string @@ -214,7 +147,7 @@ var PreferManifestList = distribution.WithManifestMediaTypes([]string{ }) // AllManifests returns all non-list manifests, the list manifest (if any), the digest the from refers to, or an error. -func AllManifests(ctx context.Context, from imagereference.DockerImageReference, repo distribution.Repository) (map[digest.Digest]distribution.Manifest, *manifestlist.DeserializedManifestList, digest.Digest, error) { +func AllManifests(ctx context.Context, from imagereference.DockerImageReference, manifests distribution.ManifestService, repo distribution.Repository) (map[digest.Digest]distribution.Manifest, *manifestlist.DeserializedManifestList, digest.Digest, error) { var srcDigest digest.Digest if len(from.ID) > 0 { srcDigest = digest.Digest(from.ID) @@ -227,10 +160,6 @@ func AllManifests(ctx context.Context, from imagereference.DockerImageReference, } else { return nil, nil, "", fmt.Errorf("no tag or digest specified") } - manifests, err := repo.Manifests(ctx) - if err != nil { - return nil, nil, "", err - } srcManifest, err := manifests.Get(ctx, srcDigest, PreferManifestList) if err != nil { return nil, nil, "", err diff --git a/pkg/cli/image/manifest/security.go b/pkg/cli/image/manifest/security.go new file mode 100644 index 0000000000..79d8f2efc1 --- /dev/null +++ b/pkg/cli/image/manifest/security.go @@ -0,0 +1,328 @@ +package manifest + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + + "github.com/spf13/pflag" + + "github.com/docker/distribution" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + kcmdutil "k8s.io/kubectl/pkg/cmd/util" + + operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1" + operatorv1alpha1scheme "github.com/openshift/client-go/operator/clientset/versioned/scheme" + operatorv1alpha1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1alpha1" + imagereference "github.com/openshift/library-go/pkg/image/reference" + "github.com/openshift/library-go/pkg/image/registryclient" + "github.com/openshift/oc/pkg/cli/image/imagesource" + "github.com/openshift/oc/pkg/cli/image/manifest/dockercredentials" +) + +type SecurityOptions struct { + RegistryConfig string + Insecure bool + SkipVerification bool + // ImageContentSourcePolicyFile to look up alternative sources + ICSPFile string + LookupClusterICSP bool + ICSPList []operatorv1alpha1.ImageContentSourcePolicy + ICSPClientFn func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) + TryAlternativeSources bool + FileDir string + + CachedContext *registryclient.Context +} + +func (o *SecurityOptions) Bind(flags *pflag.FlagSet) { + flags.StringVarP(&o.RegistryConfig, "registry-config", "a", o.RegistryConfig, "Path to your registry credentials (defaults to ~/.docker/config.json)") + flags.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP") + flags.BoolVar(&o.SkipVerification, "skip-verification", o.SkipVerification, "Skip verifying the integrity of the retrieved content. This is not recommended, but may be necessary when importing images from older image registries. Only bypass verification if the registry is known to be trustworthy.") + flags.BoolVar(&o.LookupClusterICSP, "lookup-cluster-icsp", o.LookupClusterICSP, "If set to true, look for alternative image sources from ImageContentSourcePolicy objects in cluster, honor the ordering of those sources, and fail if an ImageContentSourcePolicy is not found in cluster. Cannot be set to true with --icsp-file") + flags.StringVar(&o.ICSPFile, "icsp-file", o.ICSPFile, "Path to an ImageContentSourcePolicy file. If set, data from this file will be used to set alternative image sources. Cannot be set together with --lookup-cluster-icsp=true.") +} + +// ReferentialHTTPClient returns an http.Client that is appropriate for accessing +// blobs referenced outside of the registry (due to the present of the URLs attribute +// in the manifest reference for a layer). +func (o *SecurityOptions) ReferentialHTTPClient() (*http.Client, error) { + regContext, err := o.Context() + if err != nil { + return nil, err + } + client := &http.Client{} + if o.Insecure { + client.Transport = regContext.InsecureTransport + } else { + client.Transport = regContext.Transport + } + return client, nil +} + +func (o *SecurityOptions) Complete(f kcmdutil.Factory) error { + if o.LookupClusterICSP && len(o.ICSPFile) > 0 { + return fmt.Errorf("cannot set both --lookup-cluster-icsp=true and --icsp-file") + } + o.ICSPClientFn = func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) { + // If ImageContentSourceFile is given, only add ImageContentSource from file, don't search cluster ICSP + if len(o.ICSPFile) != 0 { + return nil, nil + } + restConfig, err := f.ToRESTConfig() + if err != nil { + // may or may not be connected to a cluster + // don't error if can't connect + klog.V(4).Infof("did not connect to an OpenShift 4.x server, will not lookup ImageContentSourcePolicies: %v", err) + return nil, nil + } + icspClient, err := operatorv1alpha1client.NewForConfig(restConfig) + if err != nil { + // may or may not be connected to a cluster + // don't error if can't connect + klog.V(4).Infof("did not connect to an OpenShift 4.x server, will not lookup ImageContentSourcePolicies: %v", err) + return nil, nil + } + return icspClient.ImageContentSourcePolicies(), nil + } + return nil +} + +func (o *SecurityOptions) Context() (*registryclient.Context, error) { + if o.CachedContext != nil { + return o.CachedContext, nil + } + context, err := o.NewContext() + if err == nil { + o.CachedContext = context + o.CachedContext.Retries = 3 + } + return context, err +} + +func (o *SecurityOptions) NewContext() (*registryclient.Context, error) { + rt, err := rest.TransportFor(&rest.Config{}) + if err != nil { + return nil, err + } + insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}}) + if err != nil { + return nil, err + } + creds := dockercredentials.NewLocal() + if len(o.RegistryConfig) > 0 { + creds, err = dockercredentials.NewFromFile(o.RegistryConfig) + if err != nil { + return nil, fmt.Errorf("unable to load --registry-config: %v", err) + } + } + context := registryclient.NewContext(rt, insecureRT).WithCredentials(creds) + context.DisableDigestVerification = o.SkipVerification + return context, nil +} + +// AddICSPsFromCluster will lookup ICSPs from cluster. Since it's not a hard requirement to find ICSPs from cluster, logs errors rather than returning errors. +func (a *SecurityOptions) AddICSPsFromCluster() error { + icspClient, err := a.ICSPClientFn() + if err != nil { + return err + } + if a.LookupClusterICSP && icspClient == nil { + return fmt.Errorf("flag --lookup-cluster-icsp was set to true, but no method was set to find ImageContentSourcePolicy objects in cluster") + } + if icspClient != nil { + icsps, err := icspClient.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + if a.LookupClusterICSP { + return fmt.Errorf("--lookup-cluster-icsp was set to true, but did not access ImageContentSourcePolicy objects in cluster: %v", err) + } + // may or may not have access to ICSPs in cluster + // don't error if can't access ICSPs + klog.V(4).Infof("did not access any ImageContentSourcePolicies in cluster: %v", err) + } + if len(icsps.Items) == 0 { + if a.LookupClusterICSP { + return fmt.Errorf("--lookup-cluster-icsp was set to true, but no ImageContentSourcePolicy objects found in cluster: %v", err) + } + klog.V(4).Info("no ImageContentSourcePolicies found in cluster") + } + a.ICSPList = append(a.ICSPList, icsps.Items...) + } + return nil +} + +func (a *SecurityOptions) AddImageSourcePoliciesFromFile() error { + icspData, err := ioutil.ReadFile(a.ICSPFile) + if err != nil { + return fmt.Errorf("unable to read ImageContentSourceFile %s: %v", a.ICSPFile, err) + } + if len(icspData) == 0 { + return fmt.Errorf("no data found in ImageContentSourceFile %s", a.ICSPFile) + } + icspObj, err := runtime.Decode(operatorv1alpha1scheme.Codecs.UniversalDeserializer(), icspData) + if err != nil { + return fmt.Errorf("error decoding ImageContentSourcePolicy from %s: %v", a.ICSPFile, err) + } + var icsp *operatorv1alpha1.ImageContentSourcePolicy + var ok bool + if icsp, ok = icspObj.(*operatorv1alpha1.ImageContentSourcePolicy); !ok { + return fmt.Errorf("could not decode ImageContentSourcePolicy from %s", a.ICSPFile) + } + a.ICSPList = append(a.ICSPList, *icsp) + return nil +} + +func (a *SecurityOptions) AddImageSources(imageRef imagereference.DockerImageReference) ([]imagereference.DockerImageReference, error) { + var imageSources []imagereference.DockerImageReference + for _, icsp := range a.ICSPList { + repoDigestMirrors := icsp.Spec.RepositoryDigestMirrors + var sourceMatches bool + for _, rdm := range repoDigestMirrors { + rdmRef, err := imagereference.Parse(rdm.Source) + if err != nil { + return nil, err + } + if imageRef.AsRepository() == rdmRef.AsRepository() { + klog.V(2).Infof("%v RepositoryDigestMirrors source matches given image", imageRef.AsRepository()) + sourceMatches = true + } + for _, m := range rdm.Mirrors { + if sourceMatches { + klog.V(2).Infof("%v RepositoryDigestMirrors mirror added to potential ImageSourcePrefixes from ImageContentSourcePolicy", m) + mRef, err := imagereference.Parse(m) + if err != nil { + return nil, err + } + imageSources = append(imageSources, mRef) + } + } + } + } + uniqueMirrors := make([]imagereference.DockerImageReference, 0, len(imageSources)) + uniqueMap := make(map[imagereference.DockerImageReference]bool) + for _, imageSourceMirror := range imageSources { + if _, ok := uniqueMap[imageSourceMirror]; !ok { + uniqueMap[imageSourceMirror] = true + uniqueMirrors = append(uniqueMirrors, imageSourceMirror) + } + } + // make sure at least 1 imagesource + // ie, make sure the image passed is included in image sources + // this is so the user-given image ref will be tried + if len(imageSources) == 0 { + imageSources = append(imageSources, imageRef.AsRepository()) + return imageSources, nil + } + klog.V(2).Infof("Found sources: %v for image: %v", uniqueMirrors, imageRef) + return uniqueMirrors, nil +} + +func (s *SecurityOptions) PreferredImageSource(image imagereference.DockerImageReference, regContext *registryclient.Context) (imagereference.DockerImageReference, distribution.Repository, distribution.ManifestService, error) { + var ( + repo distribution.Repository + replacedImage imagereference.DockerImageReference + manifests distribution.ManifestService + err error + ) + ctx := context.TODO() + typedImageRef := imagesource.TypedImageReference{Ref: image, Type: imagesource.DestinationRegistry} + + sourceOpts := &imagesource.Options{ + FileDir: s.FileDir, + Insecure: s.Insecure, + RegistryContext: regContext, + } + if len(s.ICSPFile) == 0 && !s.LookupClusterICSP && !s.TryAlternativeSources { + repo, manifests, err = sourceOpts.RepositoryWithManifests(ctx, typedImageRef) + if err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + if repo == nil || manifests == nil { + return image, nil, nil, fmt.Errorf("unable to retrieve image manifests for %v", image.String()) + } + return image, repo, manifests, nil + } + + var altSources []imagereference.DockerImageReference + // always error if given an icsp file and don't successfully connect to image repository + if len(s.ICSPFile) > 0 { + if err = s.AddImageSourcePoliciesFromFile(); err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + altSources, err = s.AddImageSources(image) + if err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + for _, icsRef := range altSources { + replacedImage = replaceImage(icsRef, image) + // if not successful, error will be handled below + if repo, manifests, err = sourceOpts.RepositoryWithManifests(context.TODO(), imagesource.TypedImageReference{Ref: replacedImage, Type: imagesource.DestinationRegistry}); err == nil { + if repo == nil || manifests == nil { + return image, nil, nil, fmt.Errorf("unable to retrieve image manifests for %v", image.String()) + } + return replacedImage, repo, manifests, nil + } + } + } + + // always error if user passed flag to look in cluster and didn't connect to image repository + if s.LookupClusterICSP { + // now try to look for ICSPs from cluster + if s.ICSPClientFn == nil { + return imagereference.DockerImageReference{}, nil, nil, fmt.Errorf("unable to find ImageContentSourcePolicy object from cluster") + } + if err = s.AddICSPsFromCluster(); err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + altSources, err = s.AddImageSources(image) + if err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + for _, icsRef := range altSources { + replacedImage = replaceImage(icsRef, image) + repo, manifests, err = sourceOpts.RepositoryWithManifests(context.TODO(), imagesource.TypedImageReference{Ref: replacedImage, Type: imagesource.DestinationRegistry}) + if err == nil && repo != nil && manifests != nil { + return replacedImage, repo, manifests, nil + } + } + } + + // now implicitly try other sources, only if TryAlternativeSources set + if s.TryAlternativeSources { + if s.ICSPClientFn == nil { + repo, manifests, err = sourceOpts.RepositoryWithManifests(context.TODO(), typedImageRef) + if err == nil && repo != nil && manifests != nil { + return typedImageRef.Ref, repo, manifests, nil + } + } + if err = s.AddICSPsFromCluster(); err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + altSources, err = s.AddImageSources(image) + if err != nil { + return imagereference.DockerImageReference{}, nil, nil, err + } + for _, icsRef := range altSources { + replacedImage = replaceImage(icsRef, image) + repo, manifests, err = sourceOpts.RepositoryWithManifests(context.TODO(), imagesource.TypedImageReference{Ref: replacedImage, Type: imagesource.DestinationRegistry}) + if err == nil && repo != nil && manifests != nil { + return replacedImage, repo, manifests, nil + } + } + } + if err != nil { + return imagereference.DockerImageReference{}, nil, nil, fmt.Errorf("unable to connect to imagerepository %s: %v", image.String(), err) + } + return replacedImage, repo, manifests, err +} + +func replaceImage(icsRef imagereference.DockerImageReference, image imagereference.DockerImageReference) imagereference.DockerImageReference { + icsRef.ID = image.ID + icsRef.Tag = image.Tag + return icsRef +} diff --git a/pkg/cli/image/manifest/security_test.go b/pkg/cli/image/manifest/security_test.go new file mode 100644 index 0000000000..5ca97dae51 --- /dev/null +++ b/pkg/cli/image/manifest/security_test.go @@ -0,0 +1,197 @@ +package manifest + +import ( + "io/ioutil" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1" + imagereference "github.com/openshift/library-go/pkg/image/reference" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func lookupAlternate(icspList []operatorv1alpha1.ImageContentSourcePolicy, icspFile string) *SecurityOptions { + return &SecurityOptions{ + ICSPFile: icspFile, + ICSPList: icspList, + LookupClusterICSP: false, + } +} + +func icspFile(t *testing.T) string { + icspFile, err := ioutil.TempFile("/tmp", "test.*.icsp.yaml") + if err != nil { + t.Errorf("error creating test icsp file: %v", err) + } + icsp := ` +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: release +spec: + repositoryDigestMirrors: + - mirrors: + - someregistry/match/file + source: quay.io/ocp-test/release + - mirrors: + - someregistry/match/file + source: quay.io/ocp-test/another-source +` + err = ioutil.WriteFile(icspFile.Name(), []byte(icsp), 0) + if err != nil { + t.Errorf("error wriing to test icsp file: %v", err) + } + return icspFile.Name() +} + +func TestAlternativeImageSources(t *testing.T) { + tests := []struct { + name string + icspList []operatorv1alpha1.ImageContentSourcePolicy + icspFile string + image string + imageSourcesExpected []string + }{ + { + name: "multiple ICSPs", + icspList: []operatorv1alpha1.ImageContentSourcePolicy{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "release", + }, + Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{ + { + Source: "quay.io/multiple/icsps", + Mirrors: []string{ + "someregistry/somerepo/release", + }, + }, + { + Source: "quay.io/ocp-test/another-release", + Mirrors: []string{ + "someregistry/somerepo/release", + }, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "another", + }, + Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{ + { + Source: "quay.io/multiple/icsps", + Mirrors: []string{ + "anotherregistry/anotherrepo/release", + }, + }, + }, + }, + }, + }, + icspFile: "", + image: "quay.io/multiple/icsps:4.5", + imageSourcesExpected: []string{"someregistry/somerepo/release", "anotherregistry/anotherrepo/release"}, + }, + { + name: "sources match ICSP file", + icspList: []operatorv1alpha1.ImageContentSourcePolicy{}, + icspFile: icspFile(t), + image: "quay.io/ocp-test/release:4.6", + imageSourcesExpected: []string{"someregistry/match/file"}, + }, + { + name: "no match ICSP file", + icspList: []operatorv1alpha1.ImageContentSourcePolicy{}, + icspFile: icspFile(t), + image: "quay.io/passed/image:4.5", + imageSourcesExpected: []string{"quay.io/passed/image"}, + }, + { + name: "ICSP mirrors match image", + icspList: []operatorv1alpha1.ImageContentSourcePolicy{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "release", + }, + Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{ + { + Source: "quay.io/ocp-test/release", + Mirrors: []string{ + "someregistry/mirrors/match", + }, + }, + }, + }, + }, + }, + icspFile: "", + image: "quay.io/ocp-test/release:4.5", + imageSourcesExpected: []string{"someregistry/mirrors/match"}, + }, + { + name: "ICSP source matches image", + icspList: []operatorv1alpha1.ImageContentSourcePolicy{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "release", + }, + Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{ + { + Source: "quay.io/source/matches", + Mirrors: []string{ + "someregistry/somerepo/release", + }, + }, + }, + }, + }, + }, + icspFile: "", + image: "quay.io/source/matches:4.5", + imageSourcesExpected: []string{"someregistry/somerepo/release"}, + }, + { + name: "no ICSP", + icspList: nil, + icspFile: "", + image: "quay.io/ocp-test/release:4.5", + imageSourcesExpected: []string{"quay.io/ocp-test/release"}, + }, + } + for _, tt := range tests { + imageRef, err := imagereference.Parse(tt.image) + if err != nil { + t.Errorf("parsing image reference error = %v", err) + } + var expectedRefs []imagereference.DockerImageReference + for _, expected := range tt.imageSourcesExpected { + expectedRef, err := imagereference.Parse(expected) + if err != nil { + t.Errorf("parsing image reference error = %v", err) + } + expectedRefs = append(expectedRefs, expectedRef) + } + + a := lookupAlternate(tt.icspList, tt.icspFile) + if len(tt.icspFile) > 0 { + err := a.AddImageSourcePoliciesFromFile() + if err != nil { + t.Errorf("add ICSP from file error = %v", err) + } + } + altSources, err := a.AddImageSources(imageRef) + if err != nil { + t.Errorf("registry client Context error = %v", err) + } + if !reflect.DeepEqual(expectedRefs, altSources) { + t.Errorf("AddAlternativeImageSource got = %v, want %v, diff = %v", altSources, expectedRefs, cmp.Diff(altSources, expectedRefs)) + } + } +} diff --git a/pkg/cli/image/mirror/mirror.go b/pkg/cli/image/mirror/mirror.go index bc57d44497..fa90f8bc4c 100644 --- a/pkg/cli/image/mirror/mirror.go +++ b/pkg/cli/image/mirror/mirror.go @@ -142,7 +142,7 @@ func NewMirrorImageOptions(streams genericclioptions.IOStreams) *MirrorImageOpti } // NewCommandMirrorImage copies images from one location to another. -func NewCmdMirrorImage(streams genericclioptions.IOStreams) *cobra.Command { +func NewCmdMirrorImage(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewMirrorImageOptions(streams) cmd := &cobra.Command{ @@ -151,16 +151,16 @@ func NewCmdMirrorImage(streams genericclioptions.IOStreams) *cobra.Command { Long: mirrorDesc, Example: mirrorExample, Run: func(c *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(c, args)) + kcmdutil.CheckErr(o.Complete(f, c, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, } flag := cmd.Flags() - o.SecurityOptions.Bind(flag) o.FilterOptions.Bind(flag) o.ParallelOptions.Bind(flag) + o.SecurityOptions.Bind(flag) flag.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Print the actions that would be taken and exit without writing to the destinations.") flag.BoolVar(&o.ContinueOnError, "continue-on-error", o.ContinueOnError, "If an error occurs, keep going and attempt to mirror as much as possible.") @@ -178,18 +178,17 @@ func NewCmdMirrorImage(streams genericclioptions.IOStreams) *cobra.Command { return cmd } -func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *MirrorImageOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { return err } - if o.FilterOptions.IsWildcardFilter() { - o.KeepManifestList = true + if err := o.SecurityOptions.Complete(f); err != nil { + return err } - registryContext, err := o.SecurityOptions.Context() - if err != nil { - return err + if o.FilterOptions.IsWildcardFilter() { + o.KeepManifestList = true } dir := o.FileDir @@ -197,6 +196,11 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { dir = o.FromFileDir } + registryContext, err := o.SecurityOptions.Context() + if err != nil { + return err + } + opts := &imagesource.Options{ FileDir: dir, Insecure: o.SecurityOptions.Insecure, @@ -209,6 +213,7 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { if err != nil { return err } + for _, filename := range o.Filenames { mappings, err := parseFile(filename, overlap, o.In, opts.ExpandWildcard) if err != nil { @@ -230,19 +235,20 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { return nil } -func (o *MirrorImageOptions) Repository(ctx context.Context, context *registryclient.Context, ref imagesource.TypedImageReference, source bool) (distribution.Repository, error) { +func (o *MirrorImageOptions) RepositoryWithManifests(ctx context.Context, regContext *registryclient.Context, ref imagesource.TypedImageReference, source bool) (distribution.Repository, distribution.ManifestService, error) { dir := o.FileDir if len(o.FromFileDir) > 0 && source { dir = o.FromFileDir } klog.V(5).Infof("Find source=%t registry with %#v", source, ref) + opts := &imagesource.Options{ FileDir: dir, Insecure: o.SecurityOptions.Insecure, AttemptS3BucketCopy: o.AttemptS3BucketCopy, - RegistryContext: context, + RegistryContext: regContext, } - return opts.Repository(ctx, ref) + return opts.RepositoryWithManifests(ctx, ref) } func (o *MirrorImageOptions) Validate() error { @@ -430,12 +436,12 @@ type contextKey struct { func (o *MirrorImageOptions) plan() (*plan, error) { ctx := apirequest.NewContext() - context, err := o.SecurityOptions.Context() + regContext, err := o.SecurityOptions.Context() if err != nil { return nil, err } - fromContext := context.Copy() - toContext := context.Copy().WithActions("pull", "push") + fromContext := regContext.Copy() + toContext := regContext.Copy().WithActions("pull", "push") toContexts := make(map[contextKey]*registryclient.Context) tree := buildTargetTree(o.Mappings) @@ -463,16 +469,13 @@ func (o *MirrorImageOptions) plan() (*plan, error) { for name := range tree { src := tree[name] q.Queue(func(_ workqueue.Work) { - srcRepo, err := o.Repository(ctx, fromContext, src.ref, true) + newImageRef, srcRepo, manifests, err := o.SecurityOptions.PreferredImageSource(src.ref.Ref, fromContext) if err != nil { plan.AddError(retrieverError{err: fmt.Errorf("unable to connect to %s: %v", src.ref, err), src: src.ref}) return } - manifests, err := srcRepo.Manifests(ctx) - if err != nil { - plan.AddError(retrieverError{src: src.ref, err: fmt.Errorf("unable to access source image %s manifests: %v", src.ref, err)}) - return - } + src.ref.Ref = newImageRef + rq := registryWorkers[name.registry] rq.Batch(func(w workqueue.Work) { // convert source tags to digests @@ -502,6 +505,7 @@ func (o *MirrorImageOptions) plan() (*plan, error) { rq.Queue(func(w workqueue.Work) { for key := range src.digests { srcDigestString, pushTargets := key, src.digests[key] + w.Parallel(func() { // load the manifest srcDigest := godigest.Digest(srcDigestString) @@ -532,16 +536,19 @@ func (o *MirrorImageOptions) plan() (*plan, error) { } for _, dst := range pushTargets { + var toRepo distribution.Repository + var toManifests distribution.ManifestService var err error if o.DryRun { toRepo, err = imagesource.NewDryRun(dst.ref) } else { - toRepo, err = o.Repository(ctx, toContexts[contextKeyForReference(dst.ref)], dst.ref, false) - } - if err != nil { - plan.AddError(retrieverError{src: src.ref, dst: dst.ref, err: fmt.Errorf("unable to connect to %s: %v", dst.ref, err)}) - continue + newImageRef, toRepo, toManifests, err = o.SecurityOptions.PreferredImageSource(dst.ref.Ref, toContexts[contextKeyForReference(dst.ref)]) + if err != nil { + plan.AddError(retrieverError{src: src.ref, dst: dst.ref, err: fmt.Errorf("unable to connect to %s: %v", dst.ref, err)}) + continue + } + dst.ref.Ref = newImageRef } canonicalTo := toRepo.Named() @@ -550,12 +557,6 @@ func (o *MirrorImageOptions) plan() (*plan, error) { repoPlan := registryPlan.RepositoryPlan(canonicalTo.String()) blobPlan := repoPlan.Blobs(src.ref, location) - toManifests, err := toRepo.Manifests(ctx) - if err != nil { - repoPlan.AddError(retrieverError{src: src.ref, dst: dst.ref, err: fmt.Errorf("unable to access destination image %s manifests: %v", src.ref, err)}) - continue - } - var mustCopyLayers bool switch { case o.Force: