diff --git a/contrib/completions/bash/oc b/contrib/completions/bash/oc index 3036807336d3..984688e8c186 100644 --- a/contrib/completions/bash/oc +++ b/contrib/completions/bash/oc @@ -4597,6 +4597,10 @@ _oc_adm_release_extract() flags_with_completion=() flags_completion=() + flags+=("--command=") + local_nonpersistent_flags+=("--command=") + flags+=("--command-os=") + local_nonpersistent_flags+=("--command-os=") flags+=("--file=") local_nonpersistent_flags+=("--file=") flags+=("--from=") @@ -4608,6 +4612,8 @@ _oc_adm_release_extract() local_nonpersistent_flags+=("--registry-config=") flags+=("--to=") local_nonpersistent_flags+=("--to=") + flags+=("--tools") + local_nonpersistent_flags+=("--tools") flags+=("--as=") flags+=("--as-group=") flags+=("--cache-dir=") @@ -4783,17 +4789,19 @@ _oc_adm_release_new() local_nonpersistent_flags+=("--dry-run") flags+=("--exclude=") local_nonpersistent_flags+=("--exclude=") - flags+=("--filename=") - two_word_flags+=("-f") - local_nonpersistent_flags+=("--filename=") flags+=("--from-dir=") local_nonpersistent_flags+=("--from-dir=") flags+=("--from-image-stream=") local_nonpersistent_flags+=("--from-image-stream=") + flags+=("--from-image-stream-file=") + two_word_flags+=("-f") + local_nonpersistent_flags+=("--from-image-stream-file=") flags+=("--from-release=") local_nonpersistent_flags+=("--from-release=") flags+=("--include=") local_nonpersistent_flags+=("--include=") + flags+=("--mapping-file=") + local_nonpersistent_flags+=("--mapping-file=") flags+=("--max-per-registry=") local_nonpersistent_flags+=("--max-per-registry=") flags+=("--metadata=") diff --git a/contrib/completions/zsh/oc b/contrib/completions/zsh/oc index 23d8cfa0e3e8..26041911231f 100644 --- a/contrib/completions/zsh/oc +++ b/contrib/completions/zsh/oc @@ -4739,6 +4739,10 @@ _oc_adm_release_extract() flags_with_completion=() flags_completion=() + flags+=("--command=") + local_nonpersistent_flags+=("--command=") + flags+=("--command-os=") + local_nonpersistent_flags+=("--command-os=") flags+=("--file=") local_nonpersistent_flags+=("--file=") flags+=("--from=") @@ -4750,6 +4754,8 @@ _oc_adm_release_extract() local_nonpersistent_flags+=("--registry-config=") flags+=("--to=") local_nonpersistent_flags+=("--to=") + flags+=("--tools") + local_nonpersistent_flags+=("--tools") flags+=("--as=") flags+=("--as-group=") flags+=("--cache-dir=") @@ -4925,17 +4931,19 @@ _oc_adm_release_new() local_nonpersistent_flags+=("--dry-run") flags+=("--exclude=") local_nonpersistent_flags+=("--exclude=") - flags+=("--filename=") - two_word_flags+=("-f") - local_nonpersistent_flags+=("--filename=") flags+=("--from-dir=") local_nonpersistent_flags+=("--from-dir=") flags+=("--from-image-stream=") local_nonpersistent_flags+=("--from-image-stream=") + flags+=("--from-image-stream-file=") + two_word_flags+=("-f") + local_nonpersistent_flags+=("--from-image-stream-file=") flags+=("--from-release=") local_nonpersistent_flags+=("--from-release=") flags+=("--include=") local_nonpersistent_flags+=("--include=") + flags+=("--mapping-file=") + local_nonpersistent_flags+=("--mapping-file=") flags+=("--max-per-registry=") local_nonpersistent_flags+=("--max-per-registry=") flags+=("--metadata=") diff --git a/pkg/oc/cli/admin/release/extract.go b/pkg/oc/cli/admin/release/extract.go index dd267aa09034..e73b29a72036 100644 --- a/pkg/oc/cli/admin/release/extract.go +++ b/pkg/oc/cli/admin/release/extract.go @@ -8,10 +8,9 @@ import ( "time" "github.com/golang/glog" - + digest "github.com/opencontainers/go-digest" "github.com/spf13/cobra" - digest "github.com/opencontainers/go-digest" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" @@ -39,17 +38,20 @@ func NewExtract(f kcmdutil.Factory, parentName string, streams genericclioptions Long: templates.LongDesc(` Extract the contents of a release image to disk - Extracts the contents of an OpenShift update image to disk for inspection or + Extracts the contents of an OpenShift release image to disk for inspection or debugging. Update images contain manifests and metadata about the operators that must be installed on the cluster for a given version. + The --tools and --command flags allow you to extract the appropriate client binaries + for your operating system to disk. --tools will create archive files containing the + current OS tools (or, if --command-os is set to '*', all OS versions). Specifying + --command for either 'oc' or 'openshift-install' will extract the binaries directly. + Instead of extracting the manifests, you can specify --git=DIR to perform a Git checkout of the source code that comprises the release. A warning will be printed if the component is not associated with source code. The command will not perform any destructive actions on your behalf except for executing a 'git checkout' which may change the current branch. Requires 'git' to be on your path. - - Experimental: This command is under active development and may change without notice. `), Run: func(cmd *cobra.Command, args []string) { kcmdutil.CheckErr(o.Complete(f, cmd, args)) @@ -58,10 +60,16 @@ func NewExtract(f kcmdutil.Factory, parentName string, streams genericclioptions } flags := cmd.Flags() flags.StringVarP(&o.RegistryConfig, "registry-config", "a", o.RegistryConfig, "Path to your registry credentials (defaults to ~/.docker/config.json)") - flags.StringVar(&o.GitExtractDir, "git", o.GitExtractDir, "Check out the sources that created this release into the provided dir. Repos will be created at //. Requires 'git' on your path.") + flags.StringVar(&o.From, "from", o.From, "Image containing the release payload.") flags.StringVar(&o.File, "file", o.File, "Extract a single file from the payload to standard output.") flags.StringVar(&o.Directory, "to", o.Directory, "Directory to write release contents to, defaults to the current directory.") + + flags.StringVar(&o.GitExtractDir, "git", o.GitExtractDir, "Check out the sources that created this release into the provided dir. Repos will be created at //. Requires 'git' on your path.") + flags.BoolVar(&o.Tools, "tools", o.Tools, "Extract the tools archives from the release image. Implies --command=*") + + flags.StringVar(&o.Command, "command", o.Command, "Specify 'oc' or 'openshift-install' to extract the client for your operating system.") + flags.StringVar(&o.CommandOperatingSystem, "command-os", o.CommandOperatingSystem, "Override which operating system command is extracted (mac, windows, linux). You map specify '*' to extract all tool archives.") return cmd } @@ -70,6 +78,10 @@ type ExtractOptions struct { From string + Tools bool + Command string + CommandOperatingSystem string + // GitExtractDir is the path of a root directory to extract the source of a release to. GitExtractDir string @@ -118,19 +130,38 @@ func (o *ExtractOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args [ } func (o *ExtractOptions) Run() error { - if len(o.From) == 0 { - return fmt.Errorf("must specify an image containing a release payload with --from") + sources := 0 + if o.Tools { + sources++ } - if o.Directory != "." && len(o.File) > 0 { - return fmt.Errorf("only one of --to and --file may be set") + if len(o.File) > 0 { + sources++ + } + if len(o.Command) > 0 { + sources++ } - if len(o.GitExtractDir) > 0 { + sources++ + } + + switch { + case sources > 1: + return fmt.Errorf("only one of --tools, --command, --file, or --git may be specified") + case len(o.From) == 0: + return fmt.Errorf("must specify an image containing a release payload with --from") + case o.Directory != "." && len(o.File) > 0: + return fmt.Errorf("only one of --to and --file may be set") + + case len(o.GitExtractDir) > 0: return o.extractGit(o.GitExtractDir) + case o.Tools: + return o.extractTools() + case len(o.Command) > 0: + return o.extractCommand(o.Command) } dir := o.Directory - if err := os.MkdirAll(dir, 0755); err != nil { + if err := os.MkdirAll(dir, 0777); err != nil { return err } @@ -200,7 +231,7 @@ func (o *ExtractOptions) Run() error { } func (o *ExtractOptions) extractGit(dir string) error { - if err := os.MkdirAll(dir, 0750); err != nil { + if err := os.MkdirAll(dir, 0777); err != nil { return err } diff --git a/pkg/oc/cli/admin/release/extract_tools.go b/pkg/oc/cli/admin/release/extract_tools.go new file mode 100644 index 000000000000..74154442d07c --- /dev/null +++ b/pkg/oc/cli/admin/release/extract_tools.go @@ -0,0 +1,420 @@ +package release + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "fmt" + "hash" + "io" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + + "github.com/golang/glog" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/cli-runtime/pkg/genericclioptions" + + imagereference "github.com/openshift/origin/pkg/image/apis/image/reference" + "github.com/openshift/origin/pkg/oc/cli/image/extract" +) + +// extractTarget describes how a file in the release image can be extracted to disk. +type extractTarget struct { + OS string + Command string + + TargetName string + + InjectReleaseImage bool + + ArchiveFormat string + AsArchive bool + AsZip bool + + Mapping extract.Mapping +} + +// extractTools extracts all referenced commands as archives in the target dir. +func (o *ExtractOptions) extractTools() error { + return o.extractCommand("") +} + +// extractTools extracts specific commands out of images referenced by the release image. +// TODO: in the future the metadata this command contains might be loaded from the release +// image, but we must maintain compatibility with older payloads if so +func (o *ExtractOptions) extractCommand(command string) error { + // Available targets is treated as a GA API and may not be changed without backwards + // compatibility of at least N-2 releases. + availableTargets := []extractTarget{ + { + OS: "darwin", + Command: "oc", + Mapping: extract.Mapping{Image: "cli-artifacts", From: "usr/share/openshift/mac/oc"}, + + ArchiveFormat: "openshift-client-mac-%s.tar.gz", + }, + { + OS: "linux", + Command: "oc", + Mapping: extract.Mapping{Image: "cli", From: "usr/bin/oc"}, + + ArchiveFormat: "openshift-client-linux-%s.tar.gz", + }, + { + OS: "windows", + Command: "oc", + Mapping: extract.Mapping{Image: "cli-artifacts", From: "usr/share/openshift/windows/oc.exe"}, + + ArchiveFormat: "openshift-client-windows-%s.zip", + AsZip: true, + }, + { + OS: "darwin", + Command: "openshift-install", + Mapping: extract.Mapping{Image: "installer-artifacts", From: "usr/share/openshift/mac/openshift-install"}, + + InjectReleaseImage: true, + ArchiveFormat: "openshift-install-mac-%s.tar.gz", + }, + { + OS: "linux", + Command: "openshift-install", + Mapping: extract.Mapping{Image: "installer", From: "usr/bin/openshift-install"}, + + InjectReleaseImage: true, + ArchiveFormat: "openshift-install-linux-%s.tar.gz", + }, + } + + currentOS := runtime.GOOS + if len(o.CommandOperatingSystem) > 0 { + currentOS = o.CommandOperatingSystem + } + if currentOS == "mac" { + currentOS = "darwin" + } + + // select the subset of targets based on command line input + var targets []extractTarget + if len(command) > 0 { + hasCommand := false + for _, target := range availableTargets { + if target.Command != command { + continue + } + hasCommand = true + if target.OS == currentOS || currentOS == "*" { + targets = []extractTarget{target} + break + } + } + if len(targets) == 0 { + if hasCommand { + return fmt.Errorf("command %q does not support the operating system %q", o.Command, currentOS) + } + return fmt.Errorf("the supported commands are 'oc' and 'openshift-install'") + } + } else { + targets = availableTargets + for i := range targets { + targets[i].AsArchive = true + targets[i].AsZip = targets[i].OS == "windows" + } + } + + // load the release image + dir := o.Directory + infoOptions := NewInfoOptions(o.IOStreams) + release, err := infoOptions.LoadReleaseInfo(o.From, false) + if err != nil { + return err + } + releaseName := release.PreferredName() + + // resolve target image references to their pull specs + missing := sets.NewString() + var validTargets []extractTarget + for _, target := range targets { + if currentOS != "*" && target.OS != currentOS { + glog.V(2).Infof("Skipping %s, does not match current OS %s", target.ArchiveFormat, target.OS) + continue + } + spec, err := findImageSpec(release.References, target.Mapping.Image, o.From) + if err != nil { + missing.Insert(target.Mapping.Image) + continue + } + glog.V(2).Infof("Will extract %s from %s", target.Mapping.From, spec) + ref, err := imagereference.Parse(spec) + if err != nil { + return err + } + target.Mapping.Image = spec + target.Mapping.ImageRef = ref + if target.AsArchive { + target.Mapping.Name = fmt.Sprintf(target.ArchiveFormat, releaseName) + target.Mapping.To = filepath.Join(dir, target.Mapping.Name) + } else { + target.Mapping.To = filepath.Join(dir, filepath.Base(target.Mapping.From)) + target.Mapping.Name = fmt.Sprintf("%s-%s", target.OS, target.Command) + } + validTargets = append(validTargets, target) + } + + if len(validTargets) == 0 { + if len(missing) == 1 { + return fmt.Errorf("the image %q containing the desired command is not available", missing.List()[0]) + } + return fmt.Errorf("some required images are missing: %s", strings.Join(missing.List(), ", ")) + } + if len(missing) > 0 { + fmt.Fprintf(o.ErrOut, "warning: Some commands can not be extracted due to missing images: %s\n", strings.Join(missing.List(), ", ")) + } + + // will extract in parallel + opts := extract.NewOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) + opts.MaxPerRegistry = 4 + opts.RegistryConfig = o.RegistryConfig + opts.OnlyFiles = true + + // create the mapping lookup of the valid targets + var extractLock sync.Mutex + targetsByName := make(map[string]extractTarget) + for _, target := range validTargets { + targetsByName[target.Mapping.Name] = target + opts.Mappings = append(opts.Mappings, target.Mapping) + } + hashByTargetName := make(map[string]string) + + // ensure to is a directory + if err := os.MkdirAll(dir, 0777); err != nil { + return err + } + + // as each layer is extracted, take the output binary and write it to disk + opts.TarEntryCallback = func(hdr *tar.Header, layer extract.LayerInfo, r io.Reader) (bool, error) { + // ensure we don't process the same mapping twice due to programmer error + target, ok := func() (extractTarget, bool) { + extractLock.Lock() + defer extractLock.Unlock() + target, ok := targetsByName[layer.Mapping.Name] + return target, ok + }() + if !ok { + return false, fmt.Errorf("unable to find target with mapping name %s", layer.Mapping.Name) + } + + // open the file + f, err := os.OpenFile(layer.Mapping.To, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755) + if err != nil { + return false, err + } + + // if we need to write an archive, wrap the file appropriately to create a single + // entry + var w io.Writer = f + var hash hash.Hash + closeFn := func() error { return nil } + if target.AsArchive { + hash = sha256.New() + w = io.MultiWriter(hash, w) + if target.AsZip { + glog.V(2).Infof("Writing %s as a ZIP archive %s", hdr.Name, layer.Mapping.To) + zw := zip.NewWriter(w) + + zh := &zip.FileHeader{ + Method: zip.Deflate, + Name: hdr.Name, + UncompressedSize64: uint64(hdr.Size), + Modified: hdr.ModTime, + } + zh.SetMode(os.FileMode(0755)) + + fw, err := zw.CreateHeader(zh) + if err != nil { + return false, err + } + + w = fw + closeFn = func() error { return zw.Close() } + + } else { + glog.V(2).Infof("Writing %s as a tar.gz archive %s", hdr.Name, layer.Mapping.To) + gw := gzip.NewWriter(w) + tw := tar.NewWriter(gw) + + if err := tw.WriteHeader(&tar.Header{ + Name: hdr.Name, + Mode: int64(os.FileMode(0755).Perm()), + Size: hdr.Size, + Typeflag: tar.TypeReg, + ModTime: hdr.ModTime, + }); err != nil { + return false, err + } + + w = tw + closeFn = func() error { + if err := tw.Close(); err != nil { + return err + } + return gw.Close() + } + } + } + + // copy the input to disk + if target.InjectReleaseImage { + var matched bool + matched, err = copyAndReplaceReleaseImage(w, r, 64*1024, o.From) + if !matched { + fmt.Fprintf(o.ErrOut, "warning: Unable to replace release image location into %s, installer will not be locked to the correct image\n", target.TargetName) + } + } else { + _, err = io.Copy(w, r) + } + if err != nil { + closeFn() + f.Close() + os.Remove(f.Name()) + return false, err + } + + // ensure the file is written to disk + if err := closeFn(); err != nil { + return false, err + } + if err := f.Close(); err != nil { + return false, err + } + if err := os.Chtimes(f.Name(), hdr.ModTime, hdr.ModTime); err != nil { + glog.V(2).Infof("Unable to set extracted file modification time: %v", err) + } + + // calculate hashes + if hash != nil { + func() { + extractLock.Lock() + defer extractLock.Unlock() + hashByTargetName[layer.Mapping.To] = hex.EncodeToString(hash.Sum(nil)) + delete(targetsByName, layer.Mapping.Name) + }() + } + + return false, nil + } + if err := opts.Run(); err != nil { + return err + } + + // write a checksum of the tar files to disk as sha256sum.txt.asc + if len(hashByTargetName) > 0 { + var keys []string + for k := range hashByTargetName { + keys = append(keys, k) + } + sort.Strings(keys) + var lines []string + for _, k := range keys { + hash := hashByTargetName[k] + lines = append(lines, fmt.Sprintf("%s %s", hash, filepath.Base(k))) + } + filename := "sha256sum.txt.asc" + if err := ioutil.WriteFile(filepath.Join(dir, filename), []byte(strings.Join(lines, "\n")), 0644); err != nil { + return fmt.Errorf("unable to write checksum file: %v", err) + } + } + + // if we did not process some targets, report that to the user and error if necessary + if len(targetsByName) > 0 { + var missing []string + for _, target := range targetsByName { + missing = append(missing, target.Mapping.From) + } + sort.Strings(missing) + if len(missing) == 1 { + return fmt.Errorf("image did not contain %s", missing[0]) + } + return fmt.Errorf("unable to find multiple files: %s", strings.Join(missing, ", ")) + } + + return nil +} + +const ( + // installerReplacement is the location within the installer binary that we can insert our + // release payload string + installerReplacement = "\x00_RELEASE_IMAGE_LOCATION_\x00XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\x00" +) + +// copyAndReplaceReleaseImage performs a targeted replacement for binaries that contain a special marker string +// as a constant, replacing the marker with releaseImage and a NUL terminating byte. It returns true if the +// replacement was performed. +func copyAndReplaceReleaseImage(w io.Writer, r io.Reader, bufferSize int, releaseImage string) (bool, error) { + if len(releaseImage)+1 > len(installerReplacement) { + return false, fmt.Errorf("the release image pull spec is longer than the maximum replacement length for the installer binary") + } + if bufferSize < len(installerReplacement) { + return false, fmt.Errorf("the buffer size must be greater than %d bytes", len(installerReplacement)) + } + + match := []byte(installerReplacement[:len(releaseImage)+1]) + offset := 0 + max := bufferSize + buf := make([]byte, max+offset) + matched := false + + for { + n, err := io.ReadFull(r, buf[offset:]) + + // search in the buffer for the expected match + end := offset + n + if n > 0 { + index := bytes.Index(buf[:end], match) + if index != -1 { + glog.V(2).Infof("Found match at %d (len=%d, offset=%d, n=%d)", index, len(buf), offset, n) + // the replacement starts at the beginning of the match, contains the release string and a terminating NUL byte + copy(buf[index:index+len(releaseImage)], []byte(releaseImage)) + buf[index+len(releaseImage)] = 0x00 + matched = true + } + } + + // write everything that we have already searched (excluding the end of the buffer that will + // be checked next pass) + nextOffset := end - len(installerReplacement) + if nextOffset < 0 || matched { + nextOffset = 0 + } + _, wErr := w.Write(buf[:end-nextOffset]) + if wErr != nil { + return matched, wErr + } + if err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return matched, nil + } + return matched, err + } + + // once we complete a single match, we can copy the rest of the file without processing + if matched { + _, err := io.Copy(w, r) + return matched, err + } + + // ensure the beginning of the buffer matches the end of the current buffer so that we + // can search for matches that span buffers + copy(buf[:nextOffset], buf[end-nextOffset:end]) + offset = nextOffset + } +} diff --git a/pkg/oc/cli/admin/release/extract_tools_test.go b/pkg/oc/cli/admin/release/extract_tools_test.go new file mode 100644 index 000000000000..c1ff6465a4e6 --- /dev/null +++ b/pkg/oc/cli/admin/release/extract_tools_test.go @@ -0,0 +1,99 @@ +package release + +import ( + "bytes" + "encoding/hex" + "math/rand" + "strings" + "testing" +) + +func Test_copyAndReplaceReleaseImage(t *testing.T) { + baseLen := len(installerReplacement) + tests := []struct { + name string + r *bytes.Buffer + buffer int + releaseImage string + wantIndex int + wantErr bool + }{ + {buffer: 10, wantErr: true, wantIndex: -1}, + {buffer: baseLen, wantErr: false, wantIndex: -1}, + + {releaseImage: "test:latest", r: fakeInput(1024, 0), wantIndex: 1024, name: "end of file"}, + {releaseImage: "test:latest", r: fakeInput(2*1024, 0), wantIndex: 2 * 1024}, + + {releaseImage: "test:latest", r: fakeInput(1024-1, 0, 1), wantIndex: 1024 - 1}, + {releaseImage: "test:latest", r: fakeInput(0, 1), wantIndex: 0}, + + {releaseImage: "test:latest", r: fakeInput(baseLen, 0), wantIndex: baseLen}, + {releaseImage: "test:latest", r: fakeInput(baseLen*2, 0), wantIndex: baseLen * 2}, + {releaseImage: "test:latest", r: fakeInput(baseLen-1, 0), wantIndex: baseLen - 1}, + {releaseImage: "test:latest", r: fakeInput(baseLen*2-1, 0), wantIndex: baseLen*2 - 1}, + {releaseImage: "test:latest", r: fakeInput(baseLen+1, 0), wantIndex: baseLen + 1}, + {releaseImage: "test:latest", r: fakeInput(baseLen*2+1, 0), wantIndex: baseLen*2 + 1}, + + {releaseImage: strings.Repeat("a", baseLen), wantIndex: -1, wantErr: true}, + {releaseImage: strings.Repeat("a", baseLen+1), wantIndex: -1, wantErr: true}, + + {releaseImage: strings.Repeat("a", baseLen-1), r: fakeInput(baseLen, 0), wantIndex: baseLen}, + {releaseImage: strings.Repeat("a", baseLen-2), r: fakeInput(baseLen, 0), wantIndex: baseLen}, + {releaseImage: strings.Repeat("a", baseLen-1), r: fakeInput(1, baseLen, 0), wantIndex: 1 + baseLen}, + {releaseImage: strings.Repeat("a", baseLen-2), r: fakeInput(1, 0, baseLen), wantIndex: 1}, + + {releaseImage: strings.Repeat("a", baseLen-1), r: fakeInput(baseLen*2, 0), wantIndex: baseLen * 2}, + {releaseImage: strings.Repeat("a", baseLen-2), r: fakeInput(baseLen*2, 0), wantIndex: baseLen * 2}, + {releaseImage: strings.Repeat("a", baseLen-1), r: fakeInput(1, baseLen*2, 0), wantIndex: 1 + baseLen*2}, + {releaseImage: strings.Repeat("a", baseLen-2), r: fakeInput(1, 0, baseLen*2), wantIndex: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := &bytes.Buffer{} + if tt.buffer == 0 { + tt.buffer = 1024 + } + if tt.r == nil { + tt.r = &bytes.Buffer{} + } + + src := tt.r.Bytes() + original := make([]byte, len(src)) + copy(original, src) + + got, err := copyAndReplaceReleaseImage(w, tt.r, tt.buffer, tt.releaseImage) + if (err != nil) != tt.wantErr { + t.Fatalf("copyAndReplaceReleaseImage() error = %v, wantErr %v", err, tt.wantErr) + } + if got != (tt.wantIndex != -1) { + t.Fatalf("copyAndReplaceReleaseImage() = %v, want %v", got, tt.wantIndex != -1) + } + if got { + if len(w.Bytes()) != len(original) { + t.Fatalf("mismatched lengths: %d vs %d \n%s\n%s", len(original), w.Len(), hex.Dump(original), hex.Dump(w.Bytes())) + } + index := bytes.Index(w.Bytes(), []byte(tt.releaseImage+"\x00")) + if index != tt.wantIndex { + t.Errorf("expected index %d, got index %d\n%s", tt.wantIndex, index, hex.Dump(w.Bytes())) + } + } else { + if !bytes.Equal(w.Bytes(), original) { + t.Fatalf("unexpected response body:\n%s\n%s", hex.Dump(original), hex.Dump(w.Bytes())) + } + } + }) + } +} + +func fakeInput(lengths ...int) *bytes.Buffer { + buf := &bytes.Buffer{} + for _, l := range lengths { + if l == 0 { + buf.WriteString(installerReplacement) + } else { + b := byte(rand.Intn(256)) + buf.Write(bytes.Repeat([]byte{b}, l)) + } + } + return buf +} diff --git a/pkg/oc/cli/admin/release/git.go b/pkg/oc/cli/admin/release/git.go index 292be4e30e90..2763e1d3bab7 100644 --- a/pkg/oc/cli/admin/release/git.go +++ b/pkg/oc/cli/admin/release/git.go @@ -251,7 +251,7 @@ func ensureCloneForRepo(dir string, repo string, alternateRepos []string, out, e if !os.IsNotExist(err) { return nil, err } - if err := os.MkdirAll(basePath, 0750); err != nil { + if err := os.MkdirAll(basePath, 0777); err != nil { return nil, err } } else { diff --git a/pkg/oc/cli/admin/release/new.go b/pkg/oc/cli/admin/release/new.go index eec04443257f..1355fb5a2b52 100644 --- a/pkg/oc/cli/admin/release/new.go +++ b/pkg/oc/cli/admin/release/new.go @@ -65,19 +65,23 @@ func NewRelease(f kcmdutil.Factory, parentName string, streams genericclioptions OpenShift uses long-running active management processes called "operators" to keep the cluster running and manage component lifecycle. This command - composes a set of images and operator definitions into a single update payload - that can be used to update a cluster. + composes a set of images with operator definitions into a single update payload + that can be used to install or update a cluster. Operators are expected to host the config they need to be installed to a cluster in the '/manifests' directory in their image. This command iterates over a set of operator images and extracts those manifests into a single, ordered list of Kubernetes objects that can then be iteratively updated on a cluster by the cluster version operator when it is time to perform an update. Manifest files are - renamed to '99__' by default, and an operator author that + renamed to '0000_70__' by default, and an operator author that needs to provide a global-ordered file (before or after other operators) should - prepend '0000_' to their filename, which instructs the release builder to not - assign a component prefix. Only images with the label - 'release.openshift.io/operator=true' are considered to be included. + prepend '0000_NN__' to their filename, which instructs the release builder + to not assign a component prefix. Only images in the input that have the image label + 'io.openshift.release.operator=true' will have manifests loaded. + + If an image is in the input but is not referenced by an operator's image-references + file, the image will not be included in the final release image unless + --include=NAME is provided. Mappings specified via SRC=DST positional arguments allows overriding particular operators with a specific image. For example: @@ -86,23 +90,31 @@ func NewRelease(f kcmdutil.Factory, parentName string, streams genericclioptions will override the default cluster-version-operator image with one pulled from registry.example.com. - - Experimental: This command is under active development and may change without notice. `), Example: templates.Examples(fmt.Sprintf(` # Create a release from the latest origin images and push to a DockerHub repo %[1]s new --from-image-stream=origin-v4.0 -n openshift --to-image docker.io/mycompany/myrepo:latest - `, parentName)), + + # Create a new release with updated metadata from a previous release + %[1]s new --from-release registry.svc.ci.openshift.org/openshift/origin-release:v4.0 --name 4.0.1 \ + --previous 4.0.0 --metadata ... --to-image docker.io/mycompany/myrepo:latest + + # Create a new release and override a single image + %[1]s new --from-release registry.svc.ci.openshift.org/openshift/origin-release:v4.0 \ + cli=docker.io/mycompany/cli:latest + `, parentName)), Run: func(cmd *cobra.Command, args []string) { kcmdutil.CheckErr(o.Complete(f, cmd, args)) + kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, } flags := cmd.Flags() // image inputs - flags.StringSliceVarP(&o.Filenames, "filename", "f", o.Filenames, "A file defining a mapping of input images to use to build the release") + flags.StringSliceVar(&o.MappingFilenames, "mapping-file", o.MappingFilenames, "A file defining a mapping of input images to use to build the release") flags.StringVar(&o.FromImageStream, "from-image-stream", o.FromImageStream, "Look at all tags in the provided image stream and build a release payload from them.") + flags.StringVarP(&o.FromImageStreamFile, "from-image-stream-file", "f", o.FromImageStreamFile, "Take the provided image stream on disk and build a release payload from it.") flags.StringVar(&o.FromDirectory, "from-dir", o.FromDirectory, "Use this directory as the source for the release payload.") flags.StringVar(&o.FromReleaseImage, "from-release", o.FromReleaseImage, "Use an existing release image as input.") flags.StringVar(&o.ReferenceMode, "reference-mode", o.ReferenceMode, "By default, the image reference from an image stream points to the public registry for the stream and the image digest. Pass 'source' to build references to the originating image.") @@ -142,17 +154,18 @@ func NewRelease(f kcmdutil.Factory, parentName string, streams genericclioptions type NewOptions struct { genericclioptions.IOStreams - FromDirectory string - Directory string - Filenames []string - Output string - Name string + FromDirectory string + Directory string + MappingFilenames []string + Output string + Name string FromReleaseImage string - FromImageStream string - Namespace string - ReferenceMode string + FromImageStream string + FromImageStreamFile string + Namespace string + ReferenceMode string ExtraComponentVersions string AllowedComponents []string @@ -190,7 +203,7 @@ type NewOptions struct { func (o *NewOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { overlap := make(map[string]string) var mappings []Mapping - for _, filename := range o.Filenames { + for _, filename := range o.MappingFilenames { fileMappings, err := parseFile(filename, overlap) if err != nil { return err @@ -225,6 +238,34 @@ func (o *NewOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []str return nil } +func (o *NewOptions) Validate() error { + sources := 0 + if len(o.FromImageStream) > 0 { + sources++ + } + if len(o.FromImageStreamFile) > 0 { + sources++ + } + if len(o.FromReleaseImage) > 0 { + sources++ + } + if len(o.FromDirectory) > 0 { + sources++ + } + if sources > 1 { + return fmt.Errorf("only one of --from-image-stream, --from-image-stream-file, --from-release, or --from-dir may be specified") + } + if sources == 0 { + if len(o.Mappings) == 0 { + return fmt.Errorf("must specify image mappings when no other source is defined") + } + } + if len(o.Mirror) > 0 && o.ReferenceMode != "" && o.ReferenceMode != "public" { + return fmt.Errorf("--reference-mode must be public or empty when using --mirror") + } + return nil +} + type imageData struct { Ref imagereference.DockerImageReference Config *docker10.DockerImageConfig @@ -232,18 +273,25 @@ type imageData struct { Directory string } -func findStatusTagEvent(tags []imageapi.NamedTagEventList, name string) *imageapi.TagEvent { - for _, tag := range tags { +func findStatusTagEvents(tags []imageapi.NamedTagEventList, name string) *imageapi.NamedTagEventList { + for i := range tags { + tag := &tags[i] if tag.Tag != name { continue } - if len(tag.Items) == 0 { - return nil - } - return &tag.Items[0] + return tag } return nil } + +func findStatusTagEvent(tags []imageapi.NamedTagEventList, name string) *imageapi.TagEvent { + events := findStatusTagEvents(tags, name) + if events == nil || len(events.Items) == 0 { + return nil + } + return &events.Items[0] +} + func findSpecTag(tags []imageapi.TagReference, name string) *imageapi.TagReference { for i, tag := range tags { if tag.Name != name { @@ -273,18 +321,6 @@ func (o *NewOptions) cleanup() { func (o *NewOptions) Run() error { defer o.cleanup() - if len(o.FromImageStream) > 0 && len(o.FromDirectory) > 0 { - return fmt.Errorf("only one of --from-image-stream and --from-dir may be specified") - } - if len(o.FromDirectory) == 0 && len(o.FromImageStream) == 0 && len(o.FromReleaseImage) == 0 { - if len(o.Mappings) == 0 { - return fmt.Errorf("must specify image mappings") - } - } - if len(o.Mirror) > 0 && o.ReferenceMode != "" && o.ReferenceMode != "public" { - return fmt.Errorf("--reference-mode must be public or empty when using --mirror") - } - extraComponentVersions, err := parseComponentVersionsLabel(o.ExtraComponentVersions) if err != nil { return fmt.Errorf("--component-versions is invalid: %v", err) @@ -438,16 +474,37 @@ func (o *NewOptions) Run() error { fmt.Fprintf(o.ErrOut, "info: Found %d images in release\n", len(is.Spec.Tags)) - case len(o.FromImageStream) > 0: + case len(o.FromImageStream) > 0, len(o.FromImageStreamFile) > 0: is = &imageapi.ImageStream{} is.Annotations = map[string]string{} if len(o.FromImageStream) > 0 && len(o.Namespace) > 0 { is.Annotations[annotationReleaseFromImageStream] = fmt.Sprintf("%s/%s", o.Namespace, o.FromImageStream) } - inputIS, err := o.ImageClient.ImageV1().ImageStreams(o.Namespace).Get(o.FromImageStream, metav1.GetOptions{}) - if err != nil { - return err + var inputIS *imageapi.ImageStream + if len(o.FromImageStreamFile) > 0 { + data, err := ioutil.ReadFile(o.FromImageStreamFile) + if os.IsNotExist(err) { + return err + } + if err != nil { + return fmt.Errorf("unable to read input image stream file: %v", err) + } + is := &imageapi.ImageStream{} + if err := yaml.Unmarshal(data, &is); err != nil { + return fmt.Errorf("unable to load input image stream file: %v", err) + } + if is.Kind != "ImageStream" || is.APIVersion != "image.openshift.io/v1" { + return fmt.Errorf("unrecognized input image stream file, must be an ImageStream in image.openshift.io/v1") + } + inputIS = is + + } else { + is, err := o.ImageClient.ImageV1().ImageStreams(o.Namespace).Get(o.FromImageStream, metav1.GetOptions{}) + if err != nil { + return err + } + inputIS = is } if inputIS.Annotations == nil { @@ -662,52 +719,101 @@ func resolveImageStreamTagsToReferenceMode(inputIS, is *imageapi.ImageStream, re if forceExternal && len(external) == 0 { return fmt.Errorf("only image streams or releases with public image repositories can be the source for releases when using the default --reference-mode") } - for _, tag := range inputIS.Status.Tags { - if exclude.Has(tag.Tag) { - glog.V(2).Infof("Excluded status tag %s", tag.Tag) + + externalFn := func(source, image string) string { + // filter source URLs + if len(source) > 0 && len(internal) > 0 && strings.HasPrefix(source, internal) { + glog.V(2).Infof("Can't use source %s because it points to the internal registry", source) + source = "" + } + // default to the external registry name + if (forceExternal || len(source) == 0) && len(external) > 0 { + return external + "@" + image + } + return source + } + + covered := sets.NewString() + for _, ref := range inputIS.Spec.Tags { + if exclude.Has(ref.Name) { + glog.V(2).Infof("Excluded spec tag %s", ref.Name) continue } - if len(tag.Items) == 0 { + + if ref.From != nil && ref.From.Kind == "DockerImage" { + switch from, err := imagereference.Parse(ref.From.Name); { + case err != nil: + return err + + case len(from.ID) > 0: + source := externalFn(ref.From.Name, from.ID) + if len(source) == 0 { + glog.V(2).Infof("Can't use spec tag %q because we cannot locate or calculate a source location", ref.Name) + continue + } + + ref := ref.DeepCopy() + ref.From = &corev1.ObjectReference{Kind: "DockerImage", Name: source} + is.Spec.Tags = append(is.Spec.Tags, *ref) + covered.Insert(ref.Name) + + case len(from.Tag) > 0: + tag := findStatusTagEvents(inputIS.Status.Tags, ref.Name) + if tag == nil { + continue + } + if len(tag.Items) == 0 { + for _, condition := range tag.Conditions { + if condition.Type == imageapi.ImportSuccess && condition.Status != metav1.StatusSuccess { + return fmt.Errorf("the tag %q in the source input stream has not been imported yet", tag.Tag) + } + } + continue + } + if ref.Generation != nil && *ref.Generation != tag.Items[0].Generation { + return fmt.Errorf("the tag %q in the source input stream has not been imported yet", tag.Tag) + } + if len(tag.Items[0].Image) == 0 { + return fmt.Errorf("the tag %q in the source input stream has no image id", tag.Tag) + } + + source := externalFn(tag.Items[0].DockerImageReference, tag.Items[0].Image) + ref := ref.DeepCopy() + ref.From = &corev1.ObjectReference{Kind: "DockerImage", Name: source} + is.Spec.Tags = append(is.Spec.Tags, *ref) + covered.Insert(ref.Name) + } continue } + // TODO: support ImageStreamTag and ImageStreamImage + } - // attempt to identify the source image - source := tag.Items[0].DockerImageReference - if len(tag.Items[0].Image) == 0 { - glog.V(2).Infof("Ignored tag %q because it had no image id or reference", tag.Tag) + for _, tag := range inputIS.Status.Tags { + if covered.Has(tag.Tag) { continue } - // eliminate status tag references that point to the outside - if len(source) > 0 { - if len(internal) > 0 && strings.HasPrefix(tag.Items[0].DockerImageReference, internal) { - glog.V(2).Infof("Can't use tag %q source %s because it points to the internal registry", tag.Tag, source) - source = "" - } + if exclude.Has(tag.Tag) { + glog.V(2).Infof("Excluded status tag %s", tag.Tag) + continue } - ref := findSpecTag(inputIS.Spec.Tags, tag.Tag) - if ref == nil { - ref = &imageapi.TagReference{Name: tag.Tag} - } else { - // prevent unimported images from being skipped - if ref.Generation != nil && *ref.Generation != tag.Items[0].Generation { - return fmt.Errorf("the tag %q in the source input stream has not been imported yet", tag.Tag) - } - // use the tag ref as the source - if ref.From != nil && ref.From.Kind == "DockerImage" && !strings.HasPrefix(ref.From.Name, internal) { - if from, err := imagereference.Parse(ref.From.Name); err == nil { - from.Tag = "" - from.ID = tag.Items[0].Image - source = from.Exact() - } else { - glog.V(2).Infof("Can't use tag %q from %s because it isn't a valid image reference", tag.Tag, ref.From.Name) + + // error if we haven't imported anything to this tag, or skip otherwise + if len(tag.Items) == 0 { + for _, condition := range tag.Conditions { + if condition.Type == imageapi.ImportSuccess && condition.Status != metav1.StatusSuccess { + return fmt.Errorf("the tag %q in the source input stream has not been imported yet", tag.Tag) } } - ref = ref.DeepCopy() + continue } - // default to the external registry name - if (forceExternal || len(source) == 0) && len(external) > 0 { - source = external + "@" + tag.Items[0].Image + // skip rather than error (user created a reference spec tag, then deleted it) + if len(tag.Items[0].Image) == 0 { + glog.V(2).Infof("the tag %q in the source input stream has no image id", tag.Tag) + continue } + + // attempt to identify the source image + source := externalFn(tag.Items[0].DockerImageReference, tag.Items[0].Image) if len(source) == 0 { glog.V(2).Infof("Can't use tag %q because we cannot locate or calculate a source location", tag.Tag) continue @@ -720,6 +826,7 @@ func resolveImageStreamTagsToReferenceMode(inputIS, is *imageapi.ImageStream, re sourceRef.ID = tag.Items[0].Image source = sourceRef.Exact() + ref := &imageapi.TagReference{Name: tag.Tag} ref.From = &corev1.ObjectReference{Kind: "DockerImage", Name: source} is.Spec.Tags = append(is.Spec.Tags, *ref) } @@ -825,7 +932,7 @@ func (o *NewOptions) extractManifests(is *imageapi.ImageStream, name string, met glog.V(2).Infof("Image %s has no %s label, skipping", m.ImageRef, annotationReleaseOperator) return false, nil } - if err := os.MkdirAll(dstDir, 0770); err != nil { + if err := os.MkdirAll(dstDir, 0777); err != nil { return false, err } if custom { @@ -842,7 +949,7 @@ func (o *NewOptions) extractManifests(is *imageapi.ImageStream, name string, met return err } if len(is.Spec.Tags) > 0 { - if err := os.MkdirAll(dir, 0770); err != nil { + if err := os.MkdirAll(dir, 0777); err != nil { return err } data, err := json.MarshalIndent(is, "", " ") @@ -900,7 +1007,7 @@ func (o *NewOptions) write(r io.Reader, is *imageapi.ImageStream, now time.Time) switch { case len(o.ToDir) > 0: glog.V(4).Infof("Writing release contents to directory %s", o.ToDir) - if err := os.MkdirAll(o.ToDir, 0755); err != nil { + if err := os.MkdirAll(o.ToDir, 0777); err != nil { return err } r, err := archive.DecompressStream(r) diff --git a/pkg/oc/cli/image/extract/extract.go b/pkg/oc/cli/image/extract/extract.go index 5a8fb41ff951..7d5a32c5773b 100644 --- a/pkg/oc/cli/image/extract/extract.go +++ b/pkg/oc/cli/image/extract/extract.go @@ -87,6 +87,7 @@ var ( type LayerInfo struct { Index int Descriptor distribution.Descriptor + Mapping *Mapping } // TarEntryFunc is called once per entry in the tar file. It may return @@ -435,11 +436,11 @@ func (o *Options) Run() error { var layerInfos []LayerInfo if byEntry != nil && !o.AllLayers { for i := len(filteredLayers) - 1; i >= 0; i-- { - layerInfos = append(layerInfos, LayerInfo{Index: i, Descriptor: filteredLayers[i]}) + layerInfos = append(layerInfos, LayerInfo{Index: i, Descriptor: filteredLayers[i], Mapping: &mapping}) } } else { for i := range filteredLayers { - layerInfos = append(layerInfos, LayerInfo{Index: i, Descriptor: filteredLayers[i]}) + layerInfos = append(layerInfos, LayerInfo{Index: i, Descriptor: filteredLayers[i], Mapping: &mapping}) } }