From 13f7888b593f12bbf78209ec66d039b6b7da0e06 Mon Sep 17 00:00:00 2001 From: Yannick Cote Date: Mon, 27 Sep 2021 10:02:53 -0400 Subject: [PATCH 01/11] sif: initial sif transport implementation Signed-off-by: Yannick Cote --- go.mod | 2 + go.sum | 3 + sif/internal/sif_util.go | 254 +++++++++++++++++++ sif/sif_src.go | 285 ++++++++++++++++++++++ sif/sif_transport.go | 119 +++++++++ transports/alltransports/alltransports.go | 2 + 6 files changed, 665 insertions(+) create mode 100644 sif/internal/sif_util.go create mode 100644 sif/sif_src.go create mode 100644 sif/sif_transport.go diff --git a/go.mod b/go.mod index 4a3675d60e..c2a4968ecd 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/opencontainers/selinux v1.10.0 github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 github.com/pkg/errors v0.9.1 + github.com/satori/go.uuid v1.2.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 github.com/ulikunitz/xz v0.5.10 @@ -37,6 +38,7 @@ require ( github.com/vbauerster/mpb/v7 v7.3.0 github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect github.com/xeipuuv/gojsonschema v1.2.0 + github.com/yhcote/sif v1.0.2-0.20181213102235-838fcf45c762 go.etcd.io/bbolt v1.3.6 go.opencensus.io v0.23.0 // indirect golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 diff --git a/go.sum b/go.sum index 7318028472..6764263d32 100644 --- a/go.sum +++ b/go.sum @@ -608,6 +608,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -690,6 +691,8 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yhcote/sif v1.0.2-0.20181213102235-838fcf45c762 h1:EjPCknEZriOHBGJX+bCd4tPmNWc326+Gq+yQdLJf4Kc= +github.com/yhcote/sif v1.0.2-0.20181213102235-838fcf45c762/go.mod h1:/5KfDnjXiMcvy8zFJjAwZa658WMEcdYAdh91A7ecN5o= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= diff --git a/sif/internal/sif_util.go b/sif/internal/sif_util.go new file mode 100644 index 0000000000..bbc047911f --- /dev/null +++ b/sif/internal/sif_util.go @@ -0,0 +1,254 @@ +package internal + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + uuid "github.com/satori/go.uuid" + "github.com/yhcote/sif/pkg/sif" + + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +type SifImage struct { + fimg sif.FileImage + rootfs *sif.Descriptor + deffile *sif.Descriptor + defReader *io.SectionReader + cmdlist []string + runscript *bytes.Buffer + env *sif.Descriptor + envReader *io.SectionReader + envlist []string + diffID digest.Digest +} + +func LoadSIFImage(path string) (image SifImage, err error) { + // open up the SIF file and get its header + image.fimg, err = sif.LoadContainer(path, true) + if err != nil { + return + } + + // check for a system partition and save it + image.rootfs, _, err = image.fimg.GetPartPrimSys() + if err != nil { + return SifImage{}, errors.Wrap(err, "looking up rootfs from SIF file") + } + + // look for a definition file object + searchDesc := sif.Descriptor{Datatype: sif.DataDeffile} + resultDescs, _, err := image.fimg.GetFromDescr(searchDesc) + if err == nil && resultDescs != nil { + // we assume in practice that typical SIF files don't hold multiple deffiles + image.deffile = resultDescs[0] + image.defReader = io.NewSectionReader(image.fimg.Fp, image.deffile.Fileoff, image.deffile.Filelen) + } + if err = image.generateConfig(); err != nil { + return SifImage{}, err + } + + // look for an environment variable set object + searchDesc = sif.Descriptor{Datatype: sif.DataEnvVar} + resultDescs, _, err = image.fimg.GetFromDescr(searchDesc) + if err == nil && resultDescs != nil { + // we assume in practice that typical SIF files don't hold multiple EnvVar sets + image.env = resultDescs[0] + image.envReader = io.NewSectionReader(image.fimg.Fp, image.env.Fileoff, image.env.Filelen) + } + + return image, nil +} + +func (image *SifImage) parseEnvironment(scanner *bufio.Scanner) error { + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || strings.HasPrefix(s, "#") { + continue + } + if strings.HasPrefix(s, "%") { + return nil + } + image.envlist = append(image.envlist, s) + } + if err := scanner.Err(); err != nil { + return errors.Wrap(err, "parsing environment from SIF definition file object") + } + return nil +} + +func (image *SifImage) parseRunscript(scanner *bufio.Scanner) error { + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(s, "%") { + return nil + } + image.cmdlist = append(image.cmdlist, s) + } + if err := scanner.Err(); err != nil { + return errors.Wrap(err, "parsing runscript from SIF definition file object") + } + return nil +} + +func (image *SifImage) generateRunscript() error { + base := `#!/bin/bash +` + image.runscript = bytes.NewBufferString(base) + for _, s := range image.envlist { + _, err := image.runscript.WriteString(fmt.Sprintln(s)) + if err != nil { + return errors.Wrap(err, "writing to runscript buffer") + } + } + for _, s := range image.cmdlist { + _, err := image.runscript.WriteString(fmt.Sprintln(s)) + if err != nil { + return errors.Wrap(err, "writing to runscript buffer") + } + } + return nil +} + +func (image *SifImage) generateConfig() error { + if image.deffile == nil { + image.cmdlist = append(image.cmdlist, "bash") + return nil + } + + // extract %environment/%runscript from definition file + var err error + scanner := bufio.NewScanner(image.defReader) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + again: + if s == `%environment` { + if err = image.parseEnvironment(scanner); err != nil { + return err + } + } else if s == `%runscript` { + if err = image.parseRunscript(scanner); err != nil { + return err + } + } + s = strings.TrimSpace(scanner.Text()) + if s == `%environment` || s == `%runscript` { + goto again + } + } + if err := scanner.Err(); err != nil { + return errors.Wrap(err, "reading lines from SIF definition file object") + } + + if len(image.cmdlist) == 0 && len(image.envlist) == 0 { + image.cmdlist = append(image.cmdlist, "bash") + } else { + image.generateRunscript() + image.cmdlist = []string{"/podman/runscript"} + } + + return nil +} + +func (image SifImage) GetConfig(config *imgspecv1.Image) error { + config.Config.Cmd = append(config.Config.Cmd, image.cmdlist...) + return nil +} + +func (image SifImage) UnloadSIFImage() (err error) { + err = image.fimg.UnloadContainer() + return +} + +func (image SifImage) GetSIFID() string { + return image.fimg.Header.ID.String() +} + +func (image SifImage) GetSIFArch() string { + return sif.GetGoArch(string(image.fimg.Header.Arch[:sif.HdrArchLen-1])) +} + +const squashFilename = "rootfs.squashfs" +const tarFilename = "rootfs.tar" + +func runUnSquashFSTar(tempdir string) (err error) { + script := ` +#!/bin/sh +unsquashfs -f ` + squashFilename + ` && tar --acls --xattrs -C ./squashfs-root -cpf ` + tarFilename + ` ./ +` + + if err = ioutil.WriteFile(filepath.Join(tempdir, "script"), []byte(script), 0755); err != nil { + return err + } + cmd := []string{"fakeroot", "--", "./script"} + + xcmd := exec.Command(cmd[0], cmd[1:]...) + xcmd.Stderr = os.Stderr + xcmd.Dir = tempdir + err = xcmd.Run() + return +} + +func (image *SifImage) writeRunscript(tempdir string) (err error) { + if image.runscript == nil { + return nil + } + rsPath := filepath.Join(tempdir, "squashfs-root", "podman") + if err = os.MkdirAll(rsPath, 0755); err != nil { + return + } + if err = ioutil.WriteFile(filepath.Join(rsPath, "runscript"), image.runscript.Bytes(), 0755); err != nil { + return errors.Wrap(err, "writing /podman/runscript") + } + return nil +} + +func (image SifImage) SquashFSToTarLayer(tempdir string) (tarpath string, err error) { + if _, err = image.fimg.Fp.Seek(image.rootfs.Fileoff, 0); err != nil { + return + } + f, err := os.Create(filepath.Join(tempdir, squashFilename)) + if err != nil { + return + } + defer f.Close() + if _, err = io.CopyN(f, image.fimg.Fp, image.rootfs.Filelen); err != nil { + return + } + if err = f.Sync(); err != nil { + return + } + if err = image.writeRunscript(tempdir); err != nil { + return + } + if err = runUnSquashFSTar(tempdir); err != nil { + return + } + return filepath.Join(tempdir, tarFilename), nil +} + +func createSIF() error { + cinfo := sif.CreateInfo{ + Pathname: "container.sif", + Launchstr: sif.HdrLaunch, + Sifversion: sif.HdrVersion, + ID: uuid.NewV4(), + } + + image, err := sif.CreateContainer(cinfo) + if err != nil { + return err + } + fmt.Printf("SIF: %+v\n", image) + + return nil +} diff --git a/sif/sif_src.go b/sif/sif_src.go new file mode 100644 index 0000000000..089ed34bfd --- /dev/null +++ b/sif/sif_src.go @@ -0,0 +1,285 @@ +package sifimage + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "time" + + "github.com/containers/image/v5/internal/tmpdir" + "github.com/containers/image/v5/sif/internal" + "github.com/containers/image/v5/types" + "github.com/klauspost/pgzip" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + + imgspecs "github.com/opencontainers/image-spec/specs-go" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +type sifImageSource struct { + ref sifReference + sifimg internal.SifImage + workdir string + diffID digest.Digest + diffSize int64 + blobID digest.Digest + blobSize int64 + blobTime time.Time + blobType string + blobFile string + config []byte + configID digest.Digest + configSize int64 + manifest []byte +} + +func (s *sifImageSource) getLayerInfo(tarpath string) error { + ftar, err := os.Open(tarpath) + if err != nil { + return fmt.Errorf("error opening %q for reading: %v", tarpath, err) + } + defer ftar.Close() + + diffDigester := digest.Canonical.Digester() + s.diffSize, err = io.Copy(diffDigester.Hash(), ftar) + if err != nil { + return fmt.Errorf("error reading %q: %v", tarpath, err) + } + s.diffID = diffDigester.Digest() + + return nil +} +func (s *sifImageSource) createBlob(tarpath string) error { + s.blobFile = fmt.Sprintf("%s.%s", tarpath, "gz") + fgz, err := os.Create(s.blobFile) + if err != nil { + return errors.Wrapf(err, "creating file for compressed blob") + } + defer fgz.Close() + fileinfo, err := fgz.Stat() + if err != nil { + return fmt.Errorf("error reading modtime of %q: %v", s.blobFile, err) + } + s.blobTime = fileinfo.ModTime() + + ftar, err := os.Open(tarpath) + if err != nil { + return fmt.Errorf("error opening %q for reading: %v", tarpath, err) + } + defer ftar.Close() + + writer := pgzip.NewWriter(fgz) + defer writer.Close() + _, err = io.Copy(writer, ftar) + if err != nil { + return fmt.Errorf("error compressing %q: %v", tarpath, err) + } + + return nil +} + +func (s *sifImageSource) getBlobInfo() error { + fgz, err := os.Open(s.blobFile) + if err != nil { + return fmt.Errorf("error opening %q for reading: %v", s.blobFile, err) + } + defer fgz.Close() + + blobDigester := digest.Canonical.Digester() + s.blobSize, err = io.Copy(blobDigester.Hash(), fgz) + if err != nil { + return fmt.Errorf("error reading %q: %v", s.blobFile, err) + } + s.blobID = blobDigester.Digest() + + return nil +} + +// newImageSource returns an ImageSource for reading from an existing directory. +// newImageSource extracts SIF objects and saves them in a temp directory. +func newImageSource(ctx context.Context, sys *types.SystemContext, ref sifReference) (types.ImageSource, error) { + var imgSrc sifImageSource + + sifimg, err := internal.LoadSIFImage(ref.resolvedFile) + if err != nil { + return nil, errors.Wrap(err, "loading SIF file") + } + + workdir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "sif") + if err != nil { + return nil, errors.Wrapf(err, "creating temp directory") + } + + tarpath, err := sifimg.SquashFSToTarLayer(workdir) + if err != nil { + return nil, errors.Wrapf(err, "converting rootfs from SquashFS to Tarball") + } + + // generate layer info + err = imgSrc.getLayerInfo(tarpath) + if err != nil { + return nil, errors.Wrapf(err, "gathering layer diff information") + } + + // prepare compressed blob + err = imgSrc.createBlob(tarpath) + if err != nil { + return nil, errors.Wrapf(err, "creating blob file") + } + + // generate blob info + err = imgSrc.getBlobInfo() + if err != nil { + return nil, errors.Wrapf(err, "gathering blob information") + } + + // populate the rootfs section of the config + rootfs := imgspecv1.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{imgSrc.diffID}, + } + created := imgSrc.blobTime + history := []imgspecv1.History{ + { + Created: &created, + CreatedBy: fmt.Sprintf("/bin/sh -c #(nop) ADD file:%s in %c", imgSrc.diffID.Hex(), os.PathSeparator), + Comment: "imported from SIF, uuid: " + sifimg.GetSIFID(), + }, + { + Created: &created, + CreatedBy: "/bin/sh -c #(nop) CMD [\"bash\"]", + EmptyLayer: true, + }, + } + + // build an OCI image config + var config imgspecv1.Image + config.Created = &created + config.Architecture = sifimg.GetSIFArch() + config.OS = "linux" + config.RootFS = rootfs + config.History = history + err = sifimg.GetConfig(&config) + if err != nil { + return nil, errors.Wrapf(err, "getting config elements from SIF") + } + + // Encode and digest the image configuration blob. + configBytes, err := json.Marshal(&config) + if err != nil { + return nil, fmt.Errorf("error generating configuration blob for %q: %v", ref.resolvedFile, err) + } + configID := digest.Canonical.FromBytes(configBytes) + configSize := int64(len(configBytes)) + + // Populate a manifest with the configuration blob and the SquashFS part as the single layer. + layerDescriptor := imgspecv1.Descriptor{ + Digest: imgSrc.blobID, + Size: imgSrc.blobSize, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + } + manifest := imgspecv1.Manifest{ + Versioned: imgspecs.Versioned{ + SchemaVersion: 2, + }, + Config: imgspecv1.Descriptor{ + Digest: configID, + Size: configSize, + MediaType: imgspecv1.MediaTypeImageConfig, + }, + Layers: []imgspecv1.Descriptor{layerDescriptor}, + } + manifestBytes, err := json.Marshal(&manifest) + if err != nil { + return nil, fmt.Errorf("error generating manifest for %q: %v", ref.resolvedFile, err) + } + + return &sifImageSource{ + ref: ref, + sifimg: sifimg, + workdir: workdir, + diffID: imgSrc.diffID, + diffSize: imgSrc.diffSize, + blobID: imgSrc.blobID, + blobSize: imgSrc.blobSize, + blobType: layerDescriptor.MediaType, + blobFile: imgSrc.blobFile, + config: configBytes, + configID: configID, + configSize: configSize, + manifest: manifestBytes, + }, nil +} + +// Reference returns the reference used to set up this source. +func (s *sifImageSource) Reference() types.ImageReference { + return s.ref +} + +// Close removes resources associated with an initialized ImageSource, if any. +func (s *sifImageSource) Close() error { + os.RemoveAll(s.workdir) + return s.sifimg.UnloadSIFImage() +} + +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *sifImageSource) HasThreadSafeGetBlob() bool { + return false +} + +// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). +// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. +// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. +func (s *sifImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) { + // We should only be asked about things in the manifest. Maybe the configuration blob. + if info.Digest == s.configID { + return ioutil.NopCloser(bytes.NewBuffer(s.config)), s.configSize, nil + } + if info.Digest == s.blobID { + reader, err := os.Open(s.blobFile) + if err != nil { + return nil, -1, fmt.Errorf("error opening %q: %v", s.blobFile, err) + } + return reader, s.blobSize, nil + } + return nil, -1, fmt.Errorf("no blob with digest %q found", info.Digest.String()) +} + +// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). +// It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); +// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +func (s *sifImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { + if instanceDigest != nil { + return nil, "", fmt.Errorf("manifest lists are not supported by the sif transport") + } + return s.manifest, imgspecv1.MediaTypeImageManifest, nil +} + +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (s *sifImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + if instanceDigest != nil { + return nil, fmt.Errorf("manifest lists are not supported by the sif transport") + } + return nil, nil +} + +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *sifImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + return nil, nil +} diff --git a/sif/sif_transport.go b/sif/sif_transport.go new file mode 100644 index 0000000000..d118b73a1d --- /dev/null +++ b/sif/sif_transport.go @@ -0,0 +1,119 @@ +package sifimage + +import ( + "context" + "fmt" + "strings" + + "github.com/containers/image/v5/directory/explicitfilepath" + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/image" + "github.com/containers/image/v5/transports" + "github.com/containers/image/v5/types" + "github.com/pkg/errors" +) + +func init() { + transports.Register(Transport) +} + +// Transport is an ImageTransport for SIF images. +var Transport = sifTransport{} + +type sifTransport struct{} + +// sifReference is an ImageReference for SIF images. +type sifReference struct { + file string // As specified by the user. May be relative, contain symlinks, etc. + resolvedFile string // Absolute file path with no symlinks, at least at the time of its creation. Primarily used for policy namespaces. +} + +func (t sifTransport) Name() string { + return "sif" +} + +// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an ImageReference. +func (t sifTransport) ParseReference(reference string) (types.ImageReference, error) { + return NewReference(reference) +} + +// NewReference returns an image file reference for a specified path. +func NewReference(file string) (types.ImageReference, error) { + resolved, err := explicitfilepath.ResolvePathToFullyExplicit(file) + if err != nil { + return nil, err + } + return sifReference{file: file, resolvedFile: resolved}, nil +} + +// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys +// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value). +// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion. +// scope passed to this function will not be "", that value is always allowed. +func (t sifTransport) ValidatePolicyConfigurationScope(scope string) error { + return errors.New(`sif: does not support any scopes except the default "" one`) +} + +func (ref sifReference) Transport() types.ImageTransport { + return Transport +} + +// StringWithinTransport returns a string representation of the reference, which MUST be such that +// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference. +func (ref sifReference) StringWithinTransport() string { + return fmt.Sprintf("%s", ref.file) +} + +// DockerReference returns a Docker reference associated with this reference +func (ref sifReference) DockerReference() reference.Named { + return nil +} + +// PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup. +func (ref sifReference) PolicyConfigurationIdentity() string { + return ref.resolvedFile +} + +// PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search +// for if explicit configuration for PolicyConfigurationIdentity() is not set +func (ref sifReference) PolicyConfigurationNamespaces() []string { + res := []string{} + path := ref.resolvedFile + for { + lastSlash := strings.LastIndex(path, "/") + if lastSlash == -1 || path == "/" { + break + } + res = append(res, path) + path = path[:lastSlash] + } + return res +} + +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. +// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, +// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (ref sifReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { + src, err := newImageSource(ctx, sys, ref) + if err != nil { + return nil, err + } + return image.FromSource(ctx, sys, src) +} + +// NewImageSource returns a types.ImageSource for this reference. +// The caller must call .Close() on the returned ImageSource. +func (ref sifReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) { + return newImageSource(ctx, sys, ref) +} + +func (ref sifReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) { + return nil, fmt.Errorf(`"sif:" locations can only be read from, not written to`) +} + +// DeleteImage deletes the named image from the registry, if supported. +func (ref sifReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error { + return errors.Errorf("Deleting images not implemented for sif: images") +} diff --git a/transports/alltransports/alltransports.go b/transports/alltransports/alltransports.go index 2110a091d2..0bae8b2599 100644 --- a/transports/alltransports/alltransports.go +++ b/transports/alltransports/alltransports.go @@ -12,7 +12,9 @@ import ( _ "github.com/containers/image/v5/oci/archive" _ "github.com/containers/image/v5/oci/layout" _ "github.com/containers/image/v5/openshift" + _ "github.com/containers/image/v5/sif" _ "github.com/containers/image/v5/tarball" + // The ostree transport is registered by ostree*.go // The storage transport is registered by storage*.go "github.com/containers/image/v5/transports" From 9b33dd16c3eb69f5a4dd8d460bed4af31b3a20d8 Mon Sep 17 00:00:00 2001 From: Yannick Cote Date: Fri, 12 Nov 2021 10:15:13 -0500 Subject: [PATCH 02/11] sif: bring code in Bring sif code in the repo instead of pulling it in at build time. Resolves PR code review discussion. Signed-off-by: Yannick Cote --- go.mod | 3 +- go.sum | 2 - sif/internal/sif_util.go | 2 +- sif/sif/LICENSE.md | 27 ++ sif/sif/create.go | 577 +++++++++++++++++++++++++++++++++++++++ sif/sif/init.go | 21 ++ sif/sif/load.go | 225 +++++++++++++++ sif/sif/lookup.go | 379 +++++++++++++++++++++++++ sif/sif/sif.go | 294 ++++++++++++++++++++ 9 files changed, 1525 insertions(+), 5 deletions(-) create mode 100644 sif/sif/LICENSE.md create mode 100644 sif/sif/create.go create mode 100644 sif/sif/init.go create mode 100644 sif/sif/load.go create mode 100644 sif/sif/lookup.go create mode 100644 sif/sif/sif.go diff --git a/go.mod b/go.mod index c2a4968ecd..190a3beffe 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/opencontainers/selinux v1.10.0 github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 github.com/pkg/errors v0.9.1 - github.com/satori/go.uuid v1.2.0 // indirect + github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 github.com/ulikunitz/xz v0.5.10 @@ -38,7 +38,6 @@ require ( github.com/vbauerster/mpb/v7 v7.3.0 github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect github.com/xeipuuv/gojsonschema v1.2.0 - github.com/yhcote/sif v1.0.2-0.20181213102235-838fcf45c762 go.etcd.io/bbolt v1.3.6 go.opencensus.io v0.23.0 // indirect golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 diff --git a/go.sum b/go.sum index 6764263d32..4109faedc9 100644 --- a/go.sum +++ b/go.sum @@ -691,8 +691,6 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yhcote/sif v1.0.2-0.20181213102235-838fcf45c762 h1:EjPCknEZriOHBGJX+bCd4tPmNWc326+Gq+yQdLJf4Kc= -github.com/yhcote/sif v1.0.2-0.20181213102235-838fcf45c762/go.mod h1:/5KfDnjXiMcvy8zFJjAwZa658WMEcdYAdh91A7ecN5o= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= diff --git a/sif/internal/sif_util.go b/sif/internal/sif_util.go index bbc047911f..88a5620c66 100644 --- a/sif/internal/sif_util.go +++ b/sif/internal/sif_util.go @@ -11,10 +11,10 @@ import ( "path/filepath" "strings" + "github.com/containers/image/v5/sif/sif" "github.com/opencontainers/go-digest" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" - "github.com/yhcote/sif/pkg/sif" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) diff --git a/sif/sif/LICENSE.md b/sif/sif/LICENSE.md new file mode 100644 index 0000000000..0686bf12c2 --- /dev/null +++ b/sif/sif/LICENSE.md @@ -0,0 +1,27 @@ +Copyright (c) 2018, Sylabs Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/sif/sif/create.go b/sif/sif/create.go new file mode 100644 index 0000000000..1b19799664 --- /dev/null +++ b/sif/sif/create.go @@ -0,0 +1,577 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +// Copyright (c) 2017, Yannick Cote All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package sif + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "fmt" + "io" + "os" + "os/user" + "path" + "strconv" + "time" +) + +// Find next offset aligned to block size +func nextAligned(offset int64, align int) int64 { + align64 := uint64(align) + offset64 := uint64(offset) + + if offset64%align64 != 0 { + offset64 = (offset64 & ^(align64 - 1)) + align64 + } + + return int64(offset64) +} + +// Set file pointer offset to next aligned block +func setFileOffNA(fimg *FileImage, alignment int) (int64, error) { + offset, err := fimg.Fp.Seek(0, 1) // get current position + if err != nil { + return -1, fmt.Errorf("seek() getting current file position: %s", err) + } + aligned := nextAligned(offset, alignment) + offset, err = fimg.Fp.Seek(aligned, 0) // set new position + if err != nil { + return -1, fmt.Errorf("seek() getting current file position: %s", err) + } + return offset, nil +} + +// Get current user and returns both uid and gid +func getUserIDs() (int64, int64, error) { + u, err := user.Current() + if err != nil { + return -1, -1, fmt.Errorf("getting current user info: %s", err) + } + + uid, err := strconv.Atoi(u.Uid) + if err != nil { + return -1, -1, fmt.Errorf("converting UID: %s", err) + } + + gid, err := strconv.Atoi(u.Gid) + if err != nil { + return -1, -1, fmt.Errorf("converting GID: %s", err) + } + + return int64(uid), int64(gid), nil +} + +// Fill all of the fields of a Descriptor +func fillDescriptor(fimg *FileImage, index int, input DescriptorInput) (err error) { + descr := &fimg.DescrArr[index] + + curoff, err := fimg.Fp.Seek(0, 1) + if err != nil { + return fmt.Errorf("while file pointer look at: %s", err) + } + + descr.Datatype = input.Datatype + descr.ID = uint32(index) + 1 + descr.Used = true + descr.Groupid = input.Groupid + descr.Link = input.Link + align := os.Getpagesize() + if input.Alignment != 0 { + align = input.Alignment + } + descr.Fileoff, err = setFileOffNA(fimg, align) + if err != nil { + return + } + descr.Filelen = input.Size + descr.Storelen = descr.Fileoff + descr.Filelen - curoff + descr.Ctime = time.Now().Unix() + descr.Mtime = time.Now().Unix() + descr.UID, descr.Gid, err = getUserIDs() + if err != nil { + return fmt.Errorf("filling descriptor: %s", err) + } + descr.SetName(path.Base(input.Fname)) + descr.SetExtra(input.Extra.Bytes()) + + // Check that none or only 1 primary partition is ever set + if descr.Datatype == DataPartition { + ptype, err := descr.GetPartType() + if err != nil { + return err + } + if ptype == PartPrimSys { + if fimg.PrimPartID != 0 { + return fmt.Errorf("only 1 FS data object may be a primary partition") + } + fimg.PrimPartID = descr.ID + arch, err := descr.GetArch() + if err != nil { + return err + } + copy(fimg.Header.Arch[:], arch[:]) + } + } + + return +} + +// Write new data object to the SIF file +func writeDataObject(fimg *FileImage, index int, input DescriptorInput) error { + // if we have bytes in input.data use that instead of an input file + if input.Data != nil { + if _, err := fimg.Fp.Write(input.Data); err != nil { + return fmt.Errorf("copying data object data to SIF file: %s", err) + } + } else { + if n, err := io.Copy(fimg.Fp, input.Fp); err != nil { + return fmt.Errorf("copying data object file to SIF file: %s", err) + } else if n != input.Size && input.Size != 0 { + return fmt.Errorf("short write while copying to SIF file") + } else if input.Size == 0 { + // coming in from os.Stdin (pipe) + descr := &fimg.DescrArr[index] + descr.Filelen = n + descr.SetName("pipe" + fmt.Sprint(index+1)) + } + } + + return nil +} + +// Find a free descriptor and create a memory representation for addition to the SIF file +func createDescriptor(fimg *FileImage, input DescriptorInput) (err error) { + var ( + idx int + v Descriptor + ) + + if fimg.Header.Dfree == 0 { + return fmt.Errorf("no descriptor table free entry") + } + + // look for a free entry in the descriptor table + for idx, v = range fimg.DescrArr { + if v.Used == false { + break + } + } + if int64(idx) == fimg.Header.Dtotal-1 && fimg.DescrArr[idx].Used == true { + return fmt.Errorf("no descriptor table free entry, warning: header.Dfree was > 0") + } + + // fill in SIF file descriptor + if err = fillDescriptor(fimg, idx, input); err != nil { + return + } + + // write data object associated to the descriptor in SIF file + if err = writeDataObject(fimg, idx, input); err != nil { + return fmt.Errorf("writing data object for SIF file: %s", err) + } + + // update some global header fields from adding this new descriptor + fimg.Header.Dfree-- + fimg.Header.Datalen += fimg.DescrArr[idx].Storelen + + return +} + +// Release and write the data object descriptor to backing storage (SIF container file) +func writeDescriptors(fimg *FileImage) error { + // first, move to descriptor start offset + if _, err := fimg.Fp.Seek(DescrStartOffset, 0); err != nil { + return fmt.Errorf("seeking to descriptor start offset: %s", err) + } + + for _, v := range fimg.DescrArr { + if err := binary.Write(fimg.Fp, binary.LittleEndian, v); err != nil { + return fmt.Errorf("binary writing descrtable to buf: %s", err) + } + } + fimg.Header.Descrlen = int64(binary.Size(fimg.DescrArr)) + + return nil +} + +// Write the global header to file +func writeHeader(fimg *FileImage) error { + // first, move to descriptor start offset + if _, err := fimg.Fp.Seek(0, 0); err != nil { + return fmt.Errorf("seeking to beginning of the file: %s", err) + } + + if err := binary.Write(fimg.Fp, binary.LittleEndian, fimg.Header); err != nil { + return fmt.Errorf("binary writing header to buf: %s", err) + } + + return nil +} + +// CreateContainer is responsible for the creation of a new SIF container +// file. It takes the creation information specification as input +// and produces an output file as specified in the input data. +func CreateContainer(cinfo CreateInfo) (fimg *FileImage, err error) { + fimg = &FileImage{} + fimg.DescrArr = make([]Descriptor, DescrNumEntries) + + // Prepare a fresh global header + copy(fimg.Header.Launch[:], cinfo.Launchstr) + copy(fimg.Header.Magic[:], HdrMagic) + copy(fimg.Header.Version[:], cinfo.Sifversion) + copy(fimg.Header.Arch[:], HdrArchUnknown) + copy(fimg.Header.ID[:], cinfo.ID[:]) + fimg.Header.Ctime = time.Now().Unix() + fimg.Header.Mtime = time.Now().Unix() + fimg.Header.Dfree = DescrNumEntries + fimg.Header.Dtotal = DescrNumEntries + fimg.Header.Descroff = DescrStartOffset + fimg.Header.Dataoff = DataStartOffset + + // Create container file + fimg.Fp, err = os.OpenFile(cinfo.Pathname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return nil, fmt.Errorf("container file creation failed: %s", err) + } + defer fimg.Fp.Close() + + // set file pointer to start of data section */ + if _, err = fimg.Fp.Seek(DataStartOffset, 0); err != nil { + return nil, fmt.Errorf("setting file offset pointer to DataStartOffset: %s", err) + } + + for _, v := range cinfo.InputDescr { + if err = createDescriptor(fimg, v); err != nil { + return + } + } + + // Write down the descriptor array + if err = writeDescriptors(fimg); err != nil { + return + } + + // Write down global header to file + if err = writeHeader(fimg); err != nil { + return + } + + return +} + +func zeroData(fimg *FileImage, descr *Descriptor) error { + // first, move to data object offset + if _, err := fimg.Fp.Seek(descr.Fileoff, 0); err != nil { + return fmt.Errorf("seeking to data object offset: %s", err) + } + + var zero [4096]byte + n := descr.Filelen + upbound := int64(4096) + for { + if n < 4096 { + upbound = n + } + + if _, err := fimg.Fp.Write(zero[:upbound]); err != nil { + return fmt.Errorf("writing 0's to data object") + } + n -= 4096 + if n <= 0 { + break + } + } + + return nil +} + +func resetDescriptor(fimg *FileImage, index int) error { + // If we remove the primary partition, set the global header Arch field to HdrArchUnknown + // to indicate that the SIF file doesn't include a primary partition and no dependency + // on any architecture exists. + _, idx, _ := fimg.GetPartPrimSys() + if idx == index { + fimg.PrimPartID = 0 + copy(fimg.Header.Arch[:], HdrArchUnknown) + } + + offset := fimg.Header.Descroff + int64(index)*int64(binary.Size(fimg.DescrArr[0])) + + // first, move to descriptor offset + if _, err := fimg.Fp.Seek(offset, 0); err != nil { + return fmt.Errorf("seeking to descriptor: %s", err) + } + + var emptyDesc Descriptor + if err := binary.Write(fimg.Fp, binary.LittleEndian, emptyDesc); err != nil { + return fmt.Errorf("binary writing empty descriptor: %s", err) + } + + return nil +} + +// AddObject add a new data object and its descriptor into the specified SIF file. +func (fimg *FileImage) AddObject(input DescriptorInput) error { + // set file pointer to the end of data section + if _, err := fimg.Fp.Seek(fimg.Header.Dataoff+fimg.Header.Datalen, 0); err != nil { + return fmt.Errorf("setting file offset pointer to DataStartOffset: %s", err) + } + + // create a new descriptor entry from input data + if err := createDescriptor(fimg, input); err != nil { + return err + } + + // write down the descriptor array + if err := writeDescriptors(fimg); err != nil { + return err + } + + fimg.Header.Mtime = time.Now().Unix() + // write down global header to file + if err := writeHeader(fimg); err != nil { + return err + } + + if err := fimg.Fp.Sync(); err != nil { + return fmt.Errorf("while sync'ing new data object to SIF file: %s", err) + } + + return nil +} + +// descrIsLast return true if passed descriptor's object is the last in a SIF file +func objectIsLast(fimg *FileImage, descr *Descriptor) bool { + if fimg.Filesize == descr.Fileoff+descr.Filelen { + return true + } + + return false +} + +// compactAtDescr joins data objects leading and following "descr" by compacting a SIF file +func compactAtDescr(fimg *FileImage, descr *Descriptor) error { + var prev Descriptor + + for _, v := range fimg.DescrArr { + if v.Used == false || v.ID == descr.ID { + continue + } else { + if v.Fileoff > prev.Fileoff { + prev = v + } + } + } + // make sure it's not the only used descriptor first + if prev.Used == true { + if err := fimg.Fp.Truncate(prev.Fileoff + prev.Filelen); err != nil { + return err + } + } else { + if err := fimg.Fp.Truncate(descr.Fileoff); err != nil { + return err + } + } + fimg.Header.Datalen -= descr.Storelen + return nil +} + +// DeleteObject removes data from a SIF file referred to by id. The descriptor for the +// data object is free'd and can be reused later. There's currenly 2 clean mode specified +// by flags: DelZero, to zero out the data region for security and DelCompact to +// remove and shink the file compacting the unused area. +func (fimg *FileImage) DeleteObject(id uint32, flags int) error { + descr, index, err := fimg.GetFromDescrID(id) + if err != nil { + return err + } + + switch flags { + case DelZero: + if err = zeroData(fimg, descr); err != nil { + return err + } + case DelCompact: + if objectIsLast(fimg, descr) { + if err = compactAtDescr(fimg, descr); err != nil { + return err + } + } else { + return fmt.Errorf("method (DelCompact) not implemented yet") + } + default: + if objectIsLast(fimg, descr) { + if err = compactAtDescr(fimg, descr); err != nil { + return err + } + } + } + + // update some global header fields from deleting this descriptor + fimg.Header.Dfree++ + fimg.Header.Mtime = time.Now().Unix() + + // zero out the unused descriptor + if err = resetDescriptor(fimg, index); err != nil { + return err + } + + // update global header + if err = writeHeader(fimg); err != nil { + return err + } + + if err := fimg.Fp.Sync(); err != nil { + return fmt.Errorf("while sync'ing deleted data object to SIF file: %s", err) + } + + return nil +} + +// SetPartExtra serializes the partition and fs type info into a binary buffer +func (di *DescriptorInput) SetPartExtra(fs Fstype, part Parttype, arch string) error { + extra := Partition{ + Fstype: fs, + Parttype: part, + } + if arch == HdrArchUnknown { + return fmt.Errorf("architecture not supported: %v", arch) + } + copy(extra.Arch[:], arch[:]) + + // serialize the partition data for integration with the base descriptor input + if err := binary.Write(&di.Extra, binary.LittleEndian, extra); err != nil { + return err + } + return nil +} + +// SetSignExtra serializes the hash type and the entity info into a binary buffer +func (di *DescriptorInput) SetSignExtra(hash Hashtype, entity string) error { + extra := Signature{ + Hashtype: hash, + } + + h, err := hex.DecodeString(entity) + if err != nil { + return err + } + copy(extra.Entity[:], h) + + // serialize the signature data for integration with the base descriptor input + if err := binary.Write(&di.Extra, binary.LittleEndian, extra); err != nil { + return err + } + return nil +} + +// SetName sets the byte array field "Name" to the value of string "name" +func (d *Descriptor) SetName(name string) { + copy(d.Name[:], []byte(name)) +} + +// SetExtra sets the extra byte array to a provided byte array +func (d *Descriptor) SetExtra(extra []byte) { + copy(d.Extra[:], extra) +} + +// SetPrimPart sets the specified system partition to be the primary one +func (fimg *FileImage) SetPrimPart(id uint32) error { + descr, _, err := fimg.GetFromDescrID(id) + if err != nil { + return err + } + + if descr.Datatype != DataPartition { + return fmt.Errorf("not a volume partition") + } + + ptype, err := descr.GetPartType() + if err != nil { + return err + } + + // if already primary system partition, nothing to do + if ptype == PartPrimSys { + return nil + } + + if ptype != PartSystem { + return fmt.Errorf("partition must be of system type") + } + + olddescr, _, err := fimg.GetPartPrimSys() + if err != nil && err != ErrNotFound { + return err + } + + fs, err := descr.GetFsType() + if err != nil { + return nil + } + + arch, err := descr.GetArch() + if err != nil { + return err + } + + copy(fimg.Header.Arch[:], arch[:]) + fimg.PrimPartID = descr.ID + + extra := Partition{ + Fstype: fs, + Parttype: PartPrimSys, + } + copy(extra.Arch[:], arch[:]) + + var extrabuf bytes.Buffer + if err := binary.Write(&extrabuf, binary.LittleEndian, extra); err != nil { + return err + } + descr.SetExtra(extrabuf.Bytes()) + + if olddescr != nil { + oldfs, err := olddescr.GetFsType() + if err != nil { + return nil + } + oldarch, err := olddescr.GetArch() + if err != nil { + return nil + } + + oldextra := Partition{ + Fstype: oldfs, + Parttype: PartSystem, + } + copy(oldextra.Arch[:], oldarch[:]) + + var oldextrabuf bytes.Buffer + if err := binary.Write(&oldextrabuf, binary.LittleEndian, oldextra); err != nil { + return err + } + olddescr.SetExtra(oldextrabuf.Bytes()) + } + + // write down the descriptor array + if err := writeDescriptors(fimg); err != nil { + return err + } + + fimg.Header.Mtime = time.Now().Unix() + // write down global header to file + if err := writeHeader(fimg); err != nil { + return err + } + + if err := fimg.Fp.Sync(); err != nil { + return fmt.Errorf("while sync'ing new data object to SIF file: %s", err) + } + + return nil +} diff --git a/sif/sif/init.go b/sif/sif/init.go new file mode 100644 index 0000000000..3e772d6b8a --- /dev/null +++ b/sif/sif/init.go @@ -0,0 +1,21 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +// Copyright (c) 2017, Yannick Cote All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package sif + +import ( + "bytes" + "log" +) + +var ( + sifLoggerBuf bytes.Buffer + siflog = log.New(&sifLoggerBuf, "", log.Ldate|log.Ltime|log.Lshortfile) +) + +func init() { +} diff --git a/sif/sif/load.go b/sif/sif/load.go new file mode 100644 index 0000000000..40f703d5b6 --- /dev/null +++ b/sif/sif/load.go @@ -0,0 +1,225 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +// Copyright (c) 2017, Yannick Cote All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package sif + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + "syscall" +) + +// Read the global header from the container file +func readHeader(fimg *FileImage) error { + if err := binary.Read(fimg.Reader, binary.LittleEndian, &fimg.Header); err != nil { + return fmt.Errorf("reading global header from container file: %s", err) + } + + return nil +} + +// Read the used descriptors and populate an in-memory representation of those in node list +func readDescriptors(fimg *FileImage) error { + // start by positioning us to the start of descriptors + _, err := fimg.Reader.Seek(fimg.Header.Descroff, 0) + if err != nil { + return fmt.Errorf("seek() setting to descriptors start: %s", err) + } + + // Initialize descriptor array (slice) and read them all from file + fimg.DescrArr = make([]Descriptor, fimg.Header.Dtotal) + if err := binary.Read(fimg.Reader, binary.LittleEndian, &fimg.DescrArr); err != nil { + fimg.DescrArr = nil + return fmt.Errorf("reading descriptor array from container file: %s", err) + } + + descr, _, err := fimg.GetPartPrimSys() + if err == nil { + fimg.PrimPartID = descr.ID + } + + return nil +} + +// Look at key fields from the global header to assess SIF validity. +// `runnable' checks is current container can run on host. +func isValidSif(fimg *FileImage) error { + // check various header fields + if string(fimg.Header.Magic[:HdrMagicLen-1]) != HdrMagic { + return fmt.Errorf("invalid SIF file: Magic |%s| want |%s|", fimg.Header.Magic, HdrMagic) + } + if string(fimg.Header.Version[:HdrVersionLen-1]) > HdrVersion { + return fmt.Errorf("invalid SIF file: Version %s want <= %s", fimg.Header.Version, HdrVersion) + } + + return nil +} + +// mapFile takes a file pointer and returns a slice of bytes representing the file data +func (fimg *FileImage) mapFile(rdonly bool) error { + prot := syscall.PROT_READ + flags := syscall.MAP_PRIVATE + + info, err := fimg.Fp.Stat() + if err != nil { + return fmt.Errorf("while trying to size SIF file to mmap") + } + fimg.Filesize = info.Size() + + size := nextAligned(info.Size(), syscall.Getpagesize()) + if int64(int(size)) < info.Size() { + return fmt.Errorf("file is to big to be mapped") + } + + if rdonly == false { + prot = syscall.PROT_WRITE + flags = syscall.MAP_SHARED + } + + fimg.Filedata, err = syscall.Mmap(int(fimg.Fp.Fd()), 0, int(size), prot, flags) + if err != nil { + // mmap failed, use sequential read() instead for top of file + siflog.Printf("mmap on %s failed, reading buffer sequentially...", fimg.Fp.Name()) + fimg.Filedata = make([]byte, DataStartOffset) + + // start by positioning us to the start of the file + _, err := fimg.Fp.Seek(0, 0) + if err != nil { + return fmt.Errorf("seek() setting to start of file: %s", err) + } + + if n, err := fimg.Fp.Read(fimg.Filedata); n != DataStartOffset { + return fmt.Errorf("short read while reading top of file: %v", err) + + } + fimg.Amodebuf = true + } + + // create and associate a new bytes.Reader on top of mmap'ed or buffered data from file + fimg.Reader = bytes.NewReader(fimg.Filedata) + + return nil +} + +func (fimg *FileImage) unmapFile() error { + if fimg.Amodebuf == true { + return nil + } + if err := syscall.Munmap(fimg.Filedata); err != nil { + return fmt.Errorf("while calling unmapping SIF file") + } + return nil +} + +// LoadContainer is responsible for loading a SIF container file. It takes +// the container file name, and whether the file is opened as read-only +// as arguments. +func LoadContainer(filename string, rdonly bool) (fimg FileImage, err error) { + if rdonly { // open SIF rdonly if mounting immutable partitions or inspecting the image + if fimg.Fp, err = os.Open(filename); err != nil { + return fimg, fmt.Errorf("opening(RDONLY) container file: %s", err) + } + } else { // open SIF read-write when adding and removing data objects + if fimg.Fp, err = os.OpenFile(filename, os.O_RDWR, 0644); err != nil { + return fimg, fmt.Errorf("opening(RDWR) container file: %s", err) + } + } + + // get a memory map of the SIF file + if err = fimg.mapFile(rdonly); err != nil { + return + } + + // read global header from SIF file + if err = readHeader(&fimg); err != nil { + return + } + + // validate global header + if err = isValidSif(&fimg); err != nil { + return + } + + // read descriptor array from SIF file + if err = readDescriptors(&fimg); err != nil { + return + } + + return +} + +// LoadContainerFp is responsible for loading a SIF container file. It takes +// a *os.File pointing to an opened file, and whether the file is opened as +// read-only for arguments. +func LoadContainerFp(fp *os.File, rdonly bool) (fimg FileImage, err error) { + if fp == nil { + return fimg, fmt.Errorf("provided fp for file is invalid") + } + + fimg.Fp = fp + + // get a memory map of the SIF file + if err = fimg.mapFile(rdonly); err != nil { + return + } + + // read global header from SIF file + if err = readHeader(&fimg); err != nil { + return + } + + // validate global header + if err = isValidSif(&fimg); err != nil { + return + } + + // read descriptor array from SIF file + if err = readDescriptors(&fimg); err != nil { + return + } + + return fimg, nil +} + +// LoadContainerReader is responsible for processing SIF data from a byte stream +// and extract various components like the global header, descriptors and even +// perhaps data, depending on how much is read from the source. +func LoadContainerReader(b *bytes.Reader) (fimg FileImage, err error) { + fimg.Reader = b + + // read global header from SIF file + if err = readHeader(&fimg); err != nil { + return + } + + // validate global header + if err = isValidSif(&fimg); err != nil { + return + } + + // in the case where the reader buffer doesn't include descriptor data, we + // don't return an error and DescrArr will be set to nil + readDescriptors(&fimg) + + return fimg, nil +} + +// UnloadContainer closes the SIF container file and free associated resources if needed +func (fimg *FileImage) UnloadContainer() (err error) { + // if SIF data comes from file, not a slice buffer (see LoadContainer() variants) + if fimg.Fp != nil { + if err = fimg.unmapFile(); err != nil { + return + } + if err = fimg.Fp.Close(); err != nil { + return fmt.Errorf("closing SIF file failed, corrupted: don't use: %s", err) + } + } + return +} diff --git a/sif/sif/lookup.go b/sif/sif/lookup.go new file mode 100644 index 0000000000..5208e27d2e --- /dev/null +++ b/sif/sif/lookup.go @@ -0,0 +1,379 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +// Copyright (c) 2017, Yannick Cote All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package sif + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "strings" +) + +// ErrNotFound is the code for when no search key is not found +var ErrNotFound = errors.New("no match found") + +// ErrMultValues is the code for when search key is not unique +var ErrMultValues = errors.New("lookup would return more than one match") + +// +// Methods on (fimg *FIleImage) +// + +// GetSIFArch returns the SIF arch code from go runtime arch code +func GetSIFArch(goarch string) (sifarch string) { + var ok bool + + archMap := map[string]string{ + "386": HdrArch386, + "amd64": HdrArchAMD64, + "arm": HdrArchARM, + "arm64": HdrArchARM64, + "ppc64": HdrArchPPC64, + "ppc64le": HdrArchPPC64le, + "mips": HdrArchMIPS, + "mipsle": HdrArchMIPSle, + "mips64": HdrArchMIPS64, + "mips64le": HdrArchMIPS64le, + "s390x": HdrArchS390x, + } + + if sifarch, ok = archMap[goarch]; !ok { + sifarch = HdrArchUnknown + } + return sifarch +} + +// GetGoArch returns the go runtime arch code from the SIF arch code +func GetGoArch(sifarch string) (goarch string) { + var ok bool + + archMap := map[string]string{ + HdrArch386: "386", + HdrArchAMD64: "amd64", + HdrArchARM: "arm", + HdrArchARM64: "arm64", + HdrArchPPC64: "ppc64", + HdrArchPPC64le: "ppc64le", + HdrArchMIPS: "mips", + HdrArchMIPSle: "mipsle", + HdrArchMIPS64: "mips64", + HdrArchMIPS64le: "mips64le", + HdrArchS390x: "s390x", + } + + if goarch, ok = archMap[sifarch]; !ok { + goarch = "unknown" + } + return goarch +} + +// GetHeader returns the loaded SIF global header +func (fimg *FileImage) GetHeader() *Header { + return &fimg.Header +} + +// GetFromDescrID searches for a descriptor with +func (fimg *FileImage) GetFromDescrID(id uint32) (*Descriptor, int, error) { + var match = -1 + + for i, v := range fimg.DescrArr { + if v.Used == false { + continue + } else { + if v.ID == id { + if match != -1 { + return nil, -1, ErrMultValues + } + match = i + } + } + } + + if match == -1 { + return nil, -1, ErrNotFound + } + + return &fimg.DescrArr[match], match, nil +} + +// GetPartFromGroup searches for partition descriptors inside a specific group +func (fimg *FileImage) GetPartFromGroup(groupid uint32) ([]*Descriptor, []int, error) { + var descrs []*Descriptor + var indexes []int + var count int + + for i, v := range fimg.DescrArr { + if v.Used == false { + continue + } else { + if v.Datatype == DataPartition && v.Groupid == groupid { + indexes = append(indexes, i) + descrs = append(descrs, &fimg.DescrArr[i]) + count++ + } + } + } + + if count == 0 { + return nil, nil, ErrNotFound + } + + return descrs, indexes, nil +} + +// GetSignFromGroup searches for signature descriptors inside a specific group +func (fimg *FileImage) GetSignFromGroup(groupid uint32) ([]*Descriptor, []int, error) { + var descrs []*Descriptor + var indexes []int + var count int + + for i, v := range fimg.DescrArr { + if v.Used == false { + continue + } else { + if v.Datatype == DataSignature && v.Groupid == groupid { + indexes = append(indexes, i) + descrs = append(descrs, &fimg.DescrArr[i]) + count++ + } + } + } + + if count == 0 { + return nil, nil, ErrNotFound + } + + return descrs, indexes, nil +} + +// GetFromLinkedDescr searches for descriptors that point to "id" +func (fimg *FileImage) GetFromLinkedDescr(ID uint32) ([]*Descriptor, []int, error) { + var descrs []*Descriptor + var indexes []int + var count int + + for i, v := range fimg.DescrArr { + if v.Used == false { + continue + } else { + if v.Link == ID { + indexes = append(indexes, i) + descrs = append(descrs, &fimg.DescrArr[i]) + count++ + } + } + } + + if count == 0 { + return nil, nil, ErrNotFound + } + + return descrs, indexes, nil +} + +// GetFromDescr searches for descriptors comparing all non-nil fields of a provided descriptor +func (fimg *FileImage) GetFromDescr(descr Descriptor) ([]*Descriptor, []int, error) { + var descrs []*Descriptor + var indexes []int + var count int + + for i, v := range fimg.DescrArr { + if v.Used == false { + continue + } else { + if descr.Datatype != 0 && descr.Datatype != v.Datatype { + continue + } + if descr.ID != 0 && descr.ID != v.ID { + continue + } + if descr.Groupid != 0 && descr.Groupid != v.Groupid { + continue + } + if descr.Link != 0 && descr.Link != v.Link { + continue + } + if descr.Fileoff != 0 && descr.Fileoff != v.Fileoff { + continue + } + if descr.Filelen != 0 && descr.Filelen != v.Filelen { + continue + } + if descr.Storelen != 0 && descr.Storelen != v.Storelen { + continue + } + if descr.Ctime != 0 && descr.Ctime != v.Ctime { + continue + } + if descr.Mtime != 0 && descr.Mtime != v.Mtime { + continue + } + if descr.UID != 0 && descr.UID != v.UID { + continue + } + if descr.Gid != 0 && descr.Gid != v.Gid { + continue + } + if descr.Name[0] != 0 && !bytes.Equal(descr.Name[:], v.Name[:]) { + continue + } + + indexes = append(indexes, i) + descrs = append(descrs, &fimg.DescrArr[i]) + count++ + } + } + + if count == 0 { + return nil, nil, ErrNotFound + } + + return descrs, indexes, nil +} + +// +// Methods on (descr *Descriptor) +// + +// GetData return a memory mapped byte slice mirroring the data object in a SIF file. +func (descr *Descriptor) GetData(fimg *FileImage) []byte { + if fimg.Amodebuf == true { + fimg.Fp.Seek(descr.Fileoff, 0) + data := make([]byte, descr.Filelen) + if n, _ := fimg.Fp.Read(data); int64(n) != descr.Filelen { + return nil + } + return data + } + + return fimg.Filedata[descr.Fileoff : descr.Fileoff+descr.Filelen] +} + +// GetName returns the name tag associated with the descriptor. Analogous to file name. +func (descr *Descriptor) GetName() string { + return strings.TrimRight(string(descr.Name[:]), "\000") +} + +// GetFsType extracts the Fstype field from the Extra field of a Partition Descriptor +func (descr *Descriptor) GetFsType() (Fstype, error) { + if descr.Datatype != DataPartition { + return -1, fmt.Errorf("expected DataPartition, got %v", descr.Datatype) + } + + var pinfo Partition + b := bytes.NewReader(descr.Extra[:]) + if err := binary.Read(b, binary.LittleEndian, &pinfo); err != nil { + return -1, fmt.Errorf("while extracting Partition extra info: %s", err) + } + + return pinfo.Fstype, nil +} + +// GetPartType extracts the Parttype field from the Extra field of a Partition Descriptor +func (descr *Descriptor) GetPartType() (Parttype, error) { + if descr.Datatype != DataPartition { + return -1, fmt.Errorf("expected DataPartition, got %v", descr.Datatype) + } + + var pinfo Partition + b := bytes.NewReader(descr.Extra[:]) + if err := binary.Read(b, binary.LittleEndian, &pinfo); err != nil { + return -1, fmt.Errorf("while extracting Partition extra info: %s", err) + } + + return pinfo.Parttype, nil +} + +// GetArch extracts the Arch field from the Extra field of a Partition Descriptor +func (descr *Descriptor) GetArch() ([HdrArchLen]byte, error) { + if descr.Datatype != DataPartition { + return [HdrArchLen]byte{}, fmt.Errorf("expected DataPartition, got %v", descr.Datatype) + } + + var pinfo Partition + b := bytes.NewReader(descr.Extra[:]) + if err := binary.Read(b, binary.LittleEndian, &pinfo); err != nil { + return [HdrArchLen]byte{}, fmt.Errorf("while extracting Partition extra info: %s", err) + } + + return pinfo.Arch, nil +} + +// GetHashType extracts the Hashtype field from the Extra field of a Signature Descriptor +func (descr *Descriptor) GetHashType() (Hashtype, error) { + if descr.Datatype != DataSignature { + return -1, fmt.Errorf("expected DataSignature, got %v", descr.Datatype) + } + + var sinfo Signature + b := bytes.NewReader(descr.Extra[:]) + if err := binary.Read(b, binary.LittleEndian, &sinfo); err != nil { + return -1, fmt.Errorf("while extracting Signature extra info: %s", err) + } + + return sinfo.Hashtype, nil +} + +// GetEntity extracts the signing entity field from the Extra field of a Signature Descriptor +func (descr *Descriptor) GetEntity() ([]byte, error) { + if descr.Datatype != DataSignature { + return nil, fmt.Errorf("expected DataSignature, got %v", descr.Datatype) + } + + var sinfo Signature + b := bytes.NewReader(descr.Extra[:]) + if err := binary.Read(b, binary.LittleEndian, &sinfo); err != nil { + return nil, fmt.Errorf("while extracting Signature extra info: %s", err) + } + + return sinfo.Entity[:], nil +} + +// GetEntityString returns the string version of the stored entity +func (descr *Descriptor) GetEntityString() (string, error) { + fingerprint, err := descr.GetEntity() + if err != nil { + return "", err + } + + return fmt.Sprintf("%0X", fingerprint[:20]), nil +} + +// GetPartPrimSys returns the primary system partition if present. There should +// be only one primary system partition in a SIF file. +func (fimg *FileImage) GetPartPrimSys() (*Descriptor, int, error) { + var descr *Descriptor + index := -1 + + for i, v := range fimg.DescrArr { + if v.Used == false { + continue + } else { + if v.Datatype == DataPartition { + ptype, err := v.GetPartType() + if err != nil { + return nil, -1, err + } + if ptype == PartPrimSys { + if index != -1 { + return nil, -1, ErrMultValues + } + index = i + descr = &fimg.DescrArr[i] + } + } + } + } + + if index == -1 { + return nil, -1, ErrNotFound + } + + return descr, index, nil +} diff --git a/sif/sif/sif.go b/sif/sif/sif.go new file mode 100644 index 0000000000..d05fe5ab0b --- /dev/null +++ b/sif/sif/sif.go @@ -0,0 +1,294 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +// Copyright (c) 2017, Yannick Cote All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package sif implements data structures and routines to create +// and access SIF files. +// - sif.go contains the data definition the file format. +// - create.go implements the core functionality for the creation of +// of new SIF files. +// - load.go implements the core functionality for the loading of +// existing SIF files. +// - lookup.go mostly implements search/lookup and printing routines +// and access to specific descriptor/data found in SIF container files. +package sif + +import ( + "bytes" + "github.com/satori/go.uuid" + "os" +) + +// Layout of a SIF file (example) +// +// .================================================. +// | GLOBAL HEADER: Sifheader | +// | - launch: "#!/usr/bin/env..." | +// | - magic: "SIF_MAGIC" | +// | - version: "1" | +// | - arch: "4" | +// | - uuid: b2659d4e-bd50-4ea5-bd17-eec5e54f918e | +// | - ctime: 1504657553 | +// | - mtime: 1504657653 | +// | - ndescr: 3 | +// | - descroff: 120 | --. +// | - descrlen: 432 | | +// | - dataoff: 4096 | | +// | - datalen: 619362 | | +// |------------------------------------------------| <-' +// | DESCR[0]: Sifdeffile | +// | - Sifcommon | +// | - datatype: DATA_DEFFILE | +// | - id: 1 | +// | - groupid: 1 | +// | - link: NONE | +// | - fileoff: 4096 | --. +// | - filelen: 222 | | +// |------------------------------------------------| <-----. +// | DESCR[1]: Sifpartition | | | +// | - Sifcommon | | | +// | - datatype: DATA_PARTITION | | | +// | - id: 2 | | | +// | - groupid: 1 | | | +// | - link: NONE | | | +// | - fileoff: 4318 | ----. | +// | - filelen: 618496 | | | | +// | - fstype: Squashfs | | | | +// | - parttype: System | | | | +// | - content: Linux | | | | +// |------------------------------------------------| | | | +// | DESCR[2]: Sifsignature | | | | +// | - Sifcommon | | | | +// | - datatype: DATA_SIGNATURE | | | | +// | - id: 3 | | | | +// | - groupid: NONE | | | | +// | - link: 2 | ------' +// | - fileoff: 622814 | ------. +// | - filelen: 644 | | | | +// | - hashtype: SHA384 | | | | +// | - entity: @ | | | | +// |------------------------------------------------| <-' | | +// | Definition file data | | | +// | . | | | +// | . | | | +// | . | | | +// |------------------------------------------------| <---' | +// | File system partition image | | +// | . | | +// | . | | +// | . | | +// |------------------------------------------------| <-----' +// | Signed verification data | +// | . | +// | . | +// | . | +// `================================================' + +// SIF header constants and quantities +const ( + HdrLaunch = "#!/usr/bin/env run-singularity\n" + HdrMagic = "SIF_MAGIC" // SIF identification + HdrVersion = "02" // SIF SPEC VERSION + HdrArchUnknown = "00" // Undefined/Unsupported arch + HdrArch386 = "01" // 386 (i[3-6]86) arch code + HdrArchAMD64 = "02" // AMD64 arch code + HdrArchARM = "03" // ARM arch code + HdrArchARM64 = "04" // AARCH64 arch code + HdrArchPPC64 = "05" // PowerPC 64 arch code + HdrArchPPC64le = "06" // PowerPC 64 little-endian arch code + HdrArchMIPS = "07" // MIPS arch code + HdrArchMIPSle = "08" // MIPS little-endian arch code + HdrArchMIPS64 = "09" // MIPS64 arch code + HdrArchMIPS64le = "10" // MIPS64 little-endian arch code + HdrArchS390x = "11" // IBM s390x arch code + + HdrLaunchLen = 32 // len("#!/usr/bin/env... ") + HdrMagicLen = 10 // len("SIF_MAGIC") + HdrVersionLen = 3 // len("99") + HdrArchLen = 3 // len("99") + + DescrNumEntries = 48 // the default total number of available descriptors + DescrGroupMask = 0xf0000000 // groups start at that offset + DescrUnusedGroup = DescrGroupMask // descriptor without a group + DescrDefaultGroup = DescrGroupMask | 1 // first groupid number created + DescrUnusedLink = 0 // descriptor without link to other + DescrEntityLen = 256 // len("Joe Bloe ...") + DescrNameLen = 128 // descriptor name (string identifier) + DescrMaxPrivLen = 384 // size reserved for descriptor specific data + DescrStartOffset = 4096 // where descriptors start after global header + DataStartOffset = 32768 // where data object start after descriptors +) + +// Datatype represents the different SIF data object types stored in the image +type Datatype int32 + +// List of supported SIF data types +const ( + DataDeffile Datatype = iota + 0x4001 // definition file data object + DataEnvVar // environment variables data object + DataLabels // JSON labels data object + DataPartition // file system data object + DataSignature // signing/verification data object + DataGenericJSON // generic JSON meta-data + DataGeneric // generic / raw data +) + +// Fstype represents the different SIF file system types found in partition data objects +type Fstype int32 + +// List of supported file systems +const ( + FsSquash Fstype = iota + 1 // Squashfs file system, RDONLY + FsExt3 // EXT3 file system, RDWR (deprecated) + FsImmuObj // immutable data object archive + FsRaw // raw data +) + +// Parttype represents the different SIF container partition types (system and data) +type Parttype int32 + +// List of supported partition types +const ( + PartSystem Parttype = iota + 1 // partition hosts an operating system + PartPrimSys // partition hosts the primary operating system + PartData // partition hosts data only + PartOverlay // partition hosts an overlay +) + +// Hashtype represents the different SIF hashing function types used to fingerprint data objects +type Hashtype int32 + +// List of supported hash functions +const ( + HashSHA256 Hashtype = iota + 1 + HashSHA384 + HashSHA512 + HashBLAKE2S + HashBLAKE2B +) + +// SIF data object deletation strategies +const ( + DelZero = iota + 1 // zero the data object bytes + DelCompact // free the space used by data object +) + +// Descriptor represents the SIF descriptor type +type Descriptor struct { + Datatype Datatype // informs of descriptor type + Used bool // is the descriptor in use + ID uint32 // a unique id for this data object + Groupid uint32 // object group this data object is related to + Link uint32 // special link or relation to an id or group + Fileoff int64 // offset from start of image file + Filelen int64 // length of data in file + Storelen int64 // length of data + alignment to store data in file + + Ctime int64 // image creation time + Mtime int64 // last modification time + UID int64 // system user owning the file + Gid int64 // system group owning the file + Name [DescrNameLen]byte // descriptor name (string identifier) + Extra [DescrMaxPrivLen]byte // big enough for extra data below +} + +// Deffile represents the SIF definition-file data object descriptor +type Deffile struct { +} + +// Labels represents the SIF JSON-labels data object descriptor +type Labels struct { +} + +// Envvar represents the SIF envvar data object descriptor +type Envvar struct { +} + +// Partition represents the SIF partition data object descriptor +type Partition struct { + Fstype Fstype + Parttype Parttype + Arch [HdrArchLen]byte // arch the image is built for +} + +// Signature represents the SIF signature data object descriptor +type Signature struct { + Hashtype Hashtype + Entity [DescrEntityLen]byte +} + +// GenericJSON represents the SIF generic JSON meta-data data object descriptor +type GenericJSON struct { +} + +// Generic represents the SIF generic data object descriptor +type Generic struct { +} + +// Header describes a loaded SIF file +type Header struct { + Launch [HdrLaunchLen]byte // #! shell execution line + + Magic [HdrMagicLen]byte // look for "SIF_MAGIC" + Version [HdrVersionLen]byte // SIF version + Arch [HdrArchLen]byte // arch the primary partition is built for + ID uuid.UUID // image unique identifier + + Ctime int64 // image creation time + Mtime int64 // last modification time + + Dfree int64 // # of unused data object descr. + Dtotal int64 // # of total available data object descr. + Descroff int64 // bytes into file where descs start + Descrlen int64 // bytes used by all current descriptors + Dataoff int64 // bytes into file where data starts + Datalen int64 // bytes used by all data objects +} + +// +// This section describes SIF creation/loading data structures used when +// building or opening a SIF file. Transient data not found in the final +// SIF file. Those data structures are internal. +// + +// FileImage describes the representation of a SIF file in memory +type FileImage struct { + Header Header // the loaded SIF global header + Fp *os.File // file pointer of opened SIF file + Filesize int64 // file size of the opened SIF file + Filedata []byte // the content of the opened file + Amodebuf bool // access mode: mmap = false, buffered = true + Reader *bytes.Reader // reader on top of Mapdata + DescrArr []Descriptor // slice of loaded descriptors from SIF file + PrimPartID uint32 // ID of primary system partition if present +} + +// CreateInfo wraps all SIF file creation info needed +type CreateInfo struct { + Pathname string // the end result output filename + Launchstr string // the shell run command + Sifversion string // the SIF specification version used + ID uuid.UUID // image unique identifier + InputDescr []DescriptorInput // slice of input info for descriptor creation +} + +// DescriptorInput describes the common info needed to create a data object descriptor +type DescriptorInput struct { + Datatype Datatype // datatype being harvested for new descriptor + Groupid uint32 // group to be set for new descriptor + Link uint32 // link to be set for new descriptor + Size int64 // size of the data object for the new descriptor + Alignment int // Align requirement for data object + + Fname string // file containing data associated with the new descriptor + Fp *os.File // file pointer to opened 'fname' + Data []byte // loaded data from file + + Image *FileImage // loaded SIF file in memory + Descr *Descriptor // created end result descriptor + + Extra bytes.Buffer // where specific input type store their data +} From 1757663cc4fdfc0fd4d1d40b32c66e136f0806d7 Mon Sep 17 00:00:00 2001 From: Yannick Cote Date: Wed, 17 Nov 2021 10:11:05 -0500 Subject: [PATCH 03/11] sif: limit platform to linux Signed-off-by: Yannick Cote --- sif/internal/sif_util.go | 2 ++ sif/sif/create.go | 2 ++ sif/sif/init.go | 2 ++ sif/sif/load.go | 2 ++ sif/sif/lookup.go | 2 ++ sif/sif/sif.go | 5 ++++- sif/sif_src.go | 2 ++ sif/sif_transport.go | 2 ++ transports/alltransports/alltransports.go | 2 +- transports/alltransports/sif.go | 8 ++++++++ transports/alltransports/sif_stub.go | 9 +++++++++ 11 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 transports/alltransports/sif.go create mode 100644 transports/alltransports/sif_stub.go diff --git a/sif/internal/sif_util.go b/sif/internal/sif_util.go index 88a5620c66..c347d1011a 100644 --- a/sif/internal/sif_util.go +++ b/sif/internal/sif_util.go @@ -1,3 +1,5 @@ +// +build linux + package internal import ( diff --git a/sif/sif/create.go b/sif/sif/create.go index 1b19799664..a1185e14f3 100644 --- a/sif/sif/create.go +++ b/sif/sif/create.go @@ -5,6 +5,8 @@ // LICENSE file distributed with the sources of this project regarding your // rights to use or distribute this software. +// +build linux + package sif import ( diff --git a/sif/sif/init.go b/sif/sif/init.go index 3e772d6b8a..c5c84504c7 100644 --- a/sif/sif/init.go +++ b/sif/sif/init.go @@ -5,6 +5,8 @@ // LICENSE file distributed with the sources of this project regarding your // rights to use or distribute this software. +// +build linux + package sif import ( diff --git a/sif/sif/load.go b/sif/sif/load.go index 40f703d5b6..c9b7c17ad7 100644 --- a/sif/sif/load.go +++ b/sif/sif/load.go @@ -5,6 +5,8 @@ // LICENSE file distributed with the sources of this project regarding your // rights to use or distribute this software. +// +build linux + package sif import ( diff --git a/sif/sif/lookup.go b/sif/sif/lookup.go index 5208e27d2e..a7c4253def 100644 --- a/sif/sif/lookup.go +++ b/sif/sif/lookup.go @@ -5,6 +5,8 @@ // LICENSE file distributed with the sources of this project regarding your // rights to use or distribute this software. +// +build linux + package sif import ( diff --git a/sif/sif/sif.go b/sif/sif/sif.go index d05fe5ab0b..964d7b72f7 100644 --- a/sif/sif/sif.go +++ b/sif/sif/sif.go @@ -5,6 +5,8 @@ // LICENSE file distributed with the sources of this project regarding your // rights to use or distribute this software. +// +build linux + // Package sif implements data structures and routines to create // and access SIF files. // - sif.go contains the data definition the file format. @@ -18,8 +20,9 @@ package sif import ( "bytes" - "github.com/satori/go.uuid" "os" + + uuid "github.com/satori/go.uuid" ) // Layout of a SIF file (example) diff --git a/sif/sif_src.go b/sif/sif_src.go index 089ed34bfd..92af266472 100644 --- a/sif/sif_src.go +++ b/sif/sif_src.go @@ -1,3 +1,5 @@ +// +build linux + package sifimage import ( diff --git a/sif/sif_transport.go b/sif/sif_transport.go index d118b73a1d..75752a65d5 100644 --- a/sif/sif_transport.go +++ b/sif/sif_transport.go @@ -1,3 +1,5 @@ +// +build linux + package sifimage import ( diff --git a/transports/alltransports/alltransports.go b/transports/alltransports/alltransports.go index 0bae8b2599..9dc9b4c28b 100644 --- a/transports/alltransports/alltransports.go +++ b/transports/alltransports/alltransports.go @@ -12,9 +12,9 @@ import ( _ "github.com/containers/image/v5/oci/archive" _ "github.com/containers/image/v5/oci/layout" _ "github.com/containers/image/v5/openshift" - _ "github.com/containers/image/v5/sif" _ "github.com/containers/image/v5/tarball" + // The sif transport is registered by sif*.go // The ostree transport is registered by ostree*.go // The storage transport is registered by storage*.go "github.com/containers/image/v5/transports" diff --git a/transports/alltransports/sif.go b/transports/alltransports/sif.go new file mode 100644 index 0000000000..ba348f2d16 --- /dev/null +++ b/transports/alltransports/sif.go @@ -0,0 +1,8 @@ +// +build linux + +package alltransports + +import ( + // Register the sif transport + _ "github.com/containers/image/v5/sif" +) diff --git a/transports/alltransports/sif_stub.go b/transports/alltransports/sif_stub.go new file mode 100644 index 0000000000..7715f7be0b --- /dev/null +++ b/transports/alltransports/sif_stub.go @@ -0,0 +1,9 @@ +// +build !linux + +package alltransports + +import "github.com/containers/image/v5/transports" + +func init() { + transports.Register(transports.NewStubTransport("sif")) +} From a7517b4cd82d8631a71238ad5bba587fe7ab8e79 Mon Sep 17 00:00:00 2001 From: Yannick Cote Date: Wed, 17 Nov 2021 10:35:56 -0500 Subject: [PATCH 04/11] sif: satisfy linter Signed-off-by: Yannick Cote --- sif/internal/sif_util.go | 24 +++--------------------- sif/sif/create.go | 14 +++++--------- sif/sif/load.go | 6 +++--- sif/sif/lookup.go | 19 +++++++++++-------- sif/sif_transport.go | 2 +- 5 files changed, 23 insertions(+), 42 deletions(-) diff --git a/sif/internal/sif_util.go b/sif/internal/sif_util.go index c347d1011a..7bdbdc3884 100644 --- a/sif/internal/sif_util.go +++ b/sif/internal/sif_util.go @@ -14,9 +14,7 @@ import ( "strings" "github.com/containers/image/v5/sif/sif" - "github.com/opencontainers/go-digest" "github.com/pkg/errors" - uuid "github.com/satori/go.uuid" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -31,7 +29,6 @@ type SifImage struct { env *sif.Descriptor envReader *io.SectionReader envlist []string - diffID digest.Digest } func LoadSIFImage(path string) (image SifImage, err error) { @@ -154,7 +151,9 @@ func (image *SifImage) generateConfig() error { if len(image.cmdlist) == 0 && len(image.envlist) == 0 { image.cmdlist = append(image.cmdlist, "bash") } else { - image.generateRunscript() + if err = image.generateRunscript(); err != nil { + return errors.Wrap(err, "generating runscript") + } image.cmdlist = []string{"/podman/runscript"} } @@ -237,20 +236,3 @@ func (image SifImage) SquashFSToTarLayer(tempdir string) (tarpath string, err er } return filepath.Join(tempdir, tarFilename), nil } - -func createSIF() error { - cinfo := sif.CreateInfo{ - Pathname: "container.sif", - Launchstr: sif.HdrLaunch, - Sifversion: sif.HdrVersion, - ID: uuid.NewV4(), - } - - image, err := sif.CreateContainer(cinfo) - if err != nil { - return err - } - fmt.Printf("SIF: %+v\n", image) - - return nil -} diff --git a/sif/sif/create.go b/sif/sif/create.go index a1185e14f3..b9888f43ad 100644 --- a/sif/sif/create.go +++ b/sif/sif/create.go @@ -159,11 +159,11 @@ func createDescriptor(fimg *FileImage, input DescriptorInput) (err error) { // look for a free entry in the descriptor table for idx, v = range fimg.DescrArr { - if v.Used == false { + if !v.Used { break } } - if int64(idx) == fimg.Header.Dtotal-1 && fimg.DescrArr[idx].Used == true { + if int64(idx) == fimg.Header.Dtotal-1 && fimg.DescrArr[idx].Used { return fmt.Errorf("no descriptor table free entry, warning: header.Dfree was > 0") } @@ -349,11 +349,7 @@ func (fimg *FileImage) AddObject(input DescriptorInput) error { // descrIsLast return true if passed descriptor's object is the last in a SIF file func objectIsLast(fimg *FileImage, descr *Descriptor) bool { - if fimg.Filesize == descr.Fileoff+descr.Filelen { - return true - } - - return false + return fimg.Filesize == descr.Fileoff+descr.Filelen } // compactAtDescr joins data objects leading and following "descr" by compacting a SIF file @@ -361,7 +357,7 @@ func compactAtDescr(fimg *FileImage, descr *Descriptor) error { var prev Descriptor for _, v := range fimg.DescrArr { - if v.Used == false || v.ID == descr.ID { + if !v.Used || v.ID == descr.ID { continue } else { if v.Fileoff > prev.Fileoff { @@ -370,7 +366,7 @@ func compactAtDescr(fimg *FileImage, descr *Descriptor) error { } } // make sure it's not the only used descriptor first - if prev.Used == true { + if prev.Used { if err := fimg.Fp.Truncate(prev.Fileoff + prev.Filelen); err != nil { return err } diff --git a/sif/sif/load.go b/sif/sif/load.go index c9b7c17ad7..610f5af66f 100644 --- a/sif/sif/load.go +++ b/sif/sif/load.go @@ -79,7 +79,7 @@ func (fimg *FileImage) mapFile(rdonly bool) error { return fmt.Errorf("file is to big to be mapped") } - if rdonly == false { + if !rdonly { prot = syscall.PROT_WRITE flags = syscall.MAP_SHARED } @@ -110,7 +110,7 @@ func (fimg *FileImage) mapFile(rdonly bool) error { } func (fimg *FileImage) unmapFile() error { - if fimg.Amodebuf == true { + if fimg.Amodebuf { return nil } if err := syscall.Munmap(fimg.Filedata); err != nil { @@ -207,7 +207,7 @@ func LoadContainerReader(b *bytes.Reader) (fimg FileImage, err error) { // in the case where the reader buffer doesn't include descriptor data, we // don't return an error and DescrArr will be set to nil - readDescriptors(&fimg) + _ = readDescriptors(&fimg) return fimg, nil } diff --git a/sif/sif/lookup.go b/sif/sif/lookup.go index a7c4253def..363050827b 100644 --- a/sif/sif/lookup.go +++ b/sif/sif/lookup.go @@ -85,7 +85,7 @@ func (fimg *FileImage) GetFromDescrID(id uint32) (*Descriptor, int, error) { var match = -1 for i, v := range fimg.DescrArr { - if v.Used == false { + if !v.Used { continue } else { if v.ID == id { @@ -111,7 +111,7 @@ func (fimg *FileImage) GetPartFromGroup(groupid uint32) ([]*Descriptor, []int, e var count int for i, v := range fimg.DescrArr { - if v.Used == false { + if !v.Used { continue } else { if v.Datatype == DataPartition && v.Groupid == groupid { @@ -136,7 +136,7 @@ func (fimg *FileImage) GetSignFromGroup(groupid uint32) ([]*Descriptor, []int, e var count int for i, v := range fimg.DescrArr { - if v.Used == false { + if !v.Used { continue } else { if v.Datatype == DataSignature && v.Groupid == groupid { @@ -161,7 +161,7 @@ func (fimg *FileImage) GetFromLinkedDescr(ID uint32) ([]*Descriptor, []int, erro var count int for i, v := range fimg.DescrArr { - if v.Used == false { + if !v.Used { continue } else { if v.Link == ID { @@ -186,7 +186,7 @@ func (fimg *FileImage) GetFromDescr(descr Descriptor) ([]*Descriptor, []int, err var count int for i, v := range fimg.DescrArr { - if v.Used == false { + if !v.Used { continue } else { if descr.Datatype != 0 && descr.Datatype != v.Datatype { @@ -245,8 +245,11 @@ func (fimg *FileImage) GetFromDescr(descr Descriptor) ([]*Descriptor, []int, err // GetData return a memory mapped byte slice mirroring the data object in a SIF file. func (descr *Descriptor) GetData(fimg *FileImage) []byte { - if fimg.Amodebuf == true { - fimg.Fp.Seek(descr.Fileoff, 0) + if fimg.Amodebuf { + _, err := fimg.Fp.Seek(descr.Fileoff, 0) + if err != nil { + return nil + } data := make([]byte, descr.Filelen) if n, _ := fimg.Fp.Read(data); int64(n) != descr.Filelen { return nil @@ -354,7 +357,7 @@ func (fimg *FileImage) GetPartPrimSys() (*Descriptor, int, error) { index := -1 for i, v := range fimg.DescrArr { - if v.Used == false { + if !v.Used { continue } else { if v.Datatype == DataPartition { diff --git a/sif/sif_transport.go b/sif/sif_transport.go index 75752a65d5..17814aa64a 100644 --- a/sif/sif_transport.go +++ b/sif/sif_transport.go @@ -63,7 +63,7 @@ func (ref sifReference) Transport() types.ImageTransport { // StringWithinTransport returns a string representation of the reference, which MUST be such that // reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference. func (ref sifReference) StringWithinTransport() string { - return fmt.Sprintf("%s", ref.file) + return ref.file } // DockerReference returns a Docker reference associated with this reference From 8f3f5466d23a216335c5d70d7d97266014b91a1a Mon Sep 17 00:00:00 2001 From: Adam Hughes Date: Wed, 17 Nov 2021 21:41:22 +0000 Subject: [PATCH 05/11] sif: use upstream sif module Signed-off-by: Adam Hughes --- go.mod | 8 +- go.sum | 254 ++++++++++++++++- sif/internal/sif_util.go | 42 ++- sif/sif/LICENSE.md | 27 -- sif/sif/create.go | 575 --------------------------------------- sif/sif/init.go | 23 -- sif/sif/load.go | 227 ---------------- sif/sif/lookup.go | 384 -------------------------- sif/sif/sif.go | 297 -------------------- 9 files changed, 268 insertions(+), 1569 deletions(-) delete mode 100644 sif/sif/LICENSE.md delete mode 100644 sif/sif/create.go delete mode 100644 sif/sif/init.go delete mode 100644 sif/sif/load.go delete mode 100644 sif/sif/lookup.go delete mode 100644 sif/sif/sif.go diff --git a/go.mod b/go.mod index 190a3beffe..2ea23d2f86 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/ghodss/yaml v1.0.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.7.4 // indirect github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.12 @@ -30,18 +29,17 @@ require ( github.com/opencontainers/selinux v1.10.0 github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 github.com/pkg/errors v0.9.1 - github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 + github.com/sylabs/sif/v2 v2.3.0 github.com/ulikunitz/xz v0.5.10 github.com/vbatts/tar-split v0.11.2 github.com/vbauerster/mpb/v7 v7.3.0 github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect github.com/xeipuuv/gojsonschema v1.2.0 go.etcd.io/bbolt v1.3.6 - go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 - golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 diff --git a/go.sum b/go.sum index 4109faedc9..0809f6ebe1 100644 --- a/go.sum +++ b/go.sum @@ -10,17 +10,33 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774 h1:SCbEWT58NSt7d2mcFdvxC9uyrdcTfvBbPLThhkDmXzg= github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= @@ -66,6 +82,8 @@ github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/ProtonMail/go-crypto v0.0.0-20210920160938-87db9fbc61c7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= @@ -73,13 +91,19 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -90,8 +114,10 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= @@ -119,6 +145,7 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -234,6 +261,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= @@ -275,15 +303,19 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -293,6 +325,12 @@ github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYis github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -338,6 +376,9 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -353,6 +394,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -361,6 +403,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -373,11 +416,19 @@ github.com/google/go-intervals v0.0.2/go.mod h1:MkaR3LNRfeKLPmqgJYs4E66z5InYjmCj github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -403,18 +454,34 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -423,18 +490,23 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.1.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -449,6 +521,7 @@ github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -456,16 +529,21 @@ github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= @@ -476,12 +554,20 @@ github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lL github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= @@ -507,6 +593,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -556,16 +643,20 @@ github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK9 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 h1:TnbXhKzrTOyuvWrjI8W6pcoI9XPbLHFXCdN2dtUw7Rw= github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -607,10 +698,17 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= +github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -623,21 +721,27 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -652,6 +756,10 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/sylabs/release-tools v0.1.0/go.mod h1:pqP/z/11/rYMQ0OM/Nn7TxGijw7KfZwW9UolD/J1TUo= +github.com/sylabs/sif/v2 v2.3.0 h1:xkjQCdP26RMuY7y7WrhtCyZH9snqf6D4iXQGHZNRA/w= +github.com/sylabs/sif/v2 v2.3.0/go.mod h1:WQa28qH0oIRsXiexugX3EuUFbeF5Twg2OF9MoU27hzs= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= @@ -681,6 +789,8 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= +github.com/xanzy/ssh-agent v0.3.1/go.mod h1:QIE4lCeL7nkC25x+yA3LBIYfwCc1TFziCtG7cBAac6w= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b h1:6cLsL+2FW6dRAdl5iMtHgRogVCff0QpRi9653YmdcJA= github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -691,8 +801,11 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= @@ -702,34 +815,47 @@ go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 h1:A/5uWzF44DlIgdm/PQFwfMkW0JX+cIcQi/SwLAmZP5M= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -752,6 +878,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -760,11 +888,16 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -777,6 +910,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -788,35 +922,56 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b h1:SXy8Ld8oKlcogOvUAh0J5Pm5RKzgYBMMxLxt6n5XW50= +golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -859,10 +1014,17 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -873,15 +1035,24 @@ golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211001092434-39dca1131b70/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d h1:1oIt9o40TWWI9FUaveVpUvBe13FNqBNVXy3ue2fcfkw= golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= @@ -922,6 +1093,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -936,9 +1108,26 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -954,12 +1143,26 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -980,10 +1183,31 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20211005153810-c76a74d43a8e h1:Im71rbA1N3CbIag/PumYhQcNR8bLNmuOtRIyOnnLsT8= google.golang.org/genproto v0.0.0-20211005153810-c76a74d43a8e/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -998,10 +1222,18 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= @@ -1024,13 +1256,16 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -1038,6 +1273,7 @@ gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1048,8 +1284,9 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= @@ -1061,6 +1298,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= diff --git a/sif/internal/sif_util.go b/sif/internal/sif_util.go index 7bdbdc3884..ed3eabf948 100644 --- a/sif/internal/sif_util.go +++ b/sif/internal/sif_util.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package internal @@ -13,56 +14,54 @@ import ( "path/filepath" "strings" - "github.com/containers/image/v5/sif/sif" "github.com/pkg/errors" + "github.com/sylabs/sif/v2/pkg/sif" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) type SifImage struct { - fimg sif.FileImage - rootfs *sif.Descriptor + fimg *sif.FileImage + rootfs sif.Descriptor deffile *sif.Descriptor - defReader *io.SectionReader + defReader io.Reader cmdlist []string runscript *bytes.Buffer env *sif.Descriptor - envReader *io.SectionReader + envReader io.Reader envlist []string } func LoadSIFImage(path string) (image SifImage, err error) { // open up the SIF file and get its header - image.fimg, err = sif.LoadContainer(path, true) + image.fimg, err = sif.LoadContainerFromPath(path, sif.OptLoadWithFlag(os.O_RDONLY)) if err != nil { return } // check for a system partition and save it - image.rootfs, _, err = image.fimg.GetPartPrimSys() + image.rootfs, err = image.fimg.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys)) if err != nil { return SifImage{}, errors.Wrap(err, "looking up rootfs from SIF file") } // look for a definition file object - searchDesc := sif.Descriptor{Datatype: sif.DataDeffile} - resultDescs, _, err := image.fimg.GetFromDescr(searchDesc) - if err == nil && resultDescs != nil { + resultDesc, err := image.fimg.GetDescriptor(sif.WithDataType(sif.DataDeffile)) + if err == nil { // we assume in practice that typical SIF files don't hold multiple deffiles - image.deffile = resultDescs[0] - image.defReader = io.NewSectionReader(image.fimg.Fp, image.deffile.Fileoff, image.deffile.Filelen) + image.deffile = &resultDesc + image.defReader = resultDesc.GetReader() } if err = image.generateConfig(); err != nil { return SifImage{}, err } // look for an environment variable set object - searchDesc = sif.Descriptor{Datatype: sif.DataEnvVar} - resultDescs, _, err = image.fimg.GetFromDescr(searchDesc) - if err == nil && resultDescs != nil { + resultDesc, err = image.fimg.GetDescriptor(sif.WithDataType(sif.DataEnvVar)) + if err == nil { // we assume in practice that typical SIF files don't hold multiple EnvVar sets - image.env = resultDescs[0] - image.envReader = io.NewSectionReader(image.fimg.Fp, image.env.Fileoff, image.env.Filelen) + image.env = &resultDesc + image.envReader = resultDesc.GetReader() } return image, nil @@ -171,11 +170,11 @@ func (image SifImage) UnloadSIFImage() (err error) { } func (image SifImage) GetSIFID() string { - return image.fimg.Header.ID.String() + return image.fimg.ID() } func (image SifImage) GetSIFArch() string { - return sif.GetGoArch(string(image.fimg.Header.Arch[:sif.HdrArchLen-1])) + return image.fimg.PrimaryArch() } const squashFilename = "rootfs.squashfs" @@ -214,15 +213,12 @@ func (image *SifImage) writeRunscript(tempdir string) (err error) { } func (image SifImage) SquashFSToTarLayer(tempdir string) (tarpath string, err error) { - if _, err = image.fimg.Fp.Seek(image.rootfs.Fileoff, 0); err != nil { - return - } f, err := os.Create(filepath.Join(tempdir, squashFilename)) if err != nil { return } defer f.Close() - if _, err = io.CopyN(f, image.fimg.Fp, image.rootfs.Filelen); err != nil { + if _, err = io.CopyN(f, image.rootfs.GetReader(), image.rootfs.Size()); err != nil { return } if err = f.Sync(); err != nil { diff --git a/sif/sif/LICENSE.md b/sif/sif/LICENSE.md deleted file mode 100644 index 0686bf12c2..0000000000 --- a/sif/sif/LICENSE.md +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2018, Sylabs Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from this - software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/sif/sif/create.go b/sif/sif/create.go deleted file mode 100644 index b9888f43ad..0000000000 --- a/sif/sif/create.go +++ /dev/null @@ -1,575 +0,0 @@ -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. -// Copyright (c) 2017, Yannick Cote All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -// +build linux - -package sif - -import ( - "bytes" - "encoding/binary" - "encoding/hex" - "fmt" - "io" - "os" - "os/user" - "path" - "strconv" - "time" -) - -// Find next offset aligned to block size -func nextAligned(offset int64, align int) int64 { - align64 := uint64(align) - offset64 := uint64(offset) - - if offset64%align64 != 0 { - offset64 = (offset64 & ^(align64 - 1)) + align64 - } - - return int64(offset64) -} - -// Set file pointer offset to next aligned block -func setFileOffNA(fimg *FileImage, alignment int) (int64, error) { - offset, err := fimg.Fp.Seek(0, 1) // get current position - if err != nil { - return -1, fmt.Errorf("seek() getting current file position: %s", err) - } - aligned := nextAligned(offset, alignment) - offset, err = fimg.Fp.Seek(aligned, 0) // set new position - if err != nil { - return -1, fmt.Errorf("seek() getting current file position: %s", err) - } - return offset, nil -} - -// Get current user and returns both uid and gid -func getUserIDs() (int64, int64, error) { - u, err := user.Current() - if err != nil { - return -1, -1, fmt.Errorf("getting current user info: %s", err) - } - - uid, err := strconv.Atoi(u.Uid) - if err != nil { - return -1, -1, fmt.Errorf("converting UID: %s", err) - } - - gid, err := strconv.Atoi(u.Gid) - if err != nil { - return -1, -1, fmt.Errorf("converting GID: %s", err) - } - - return int64(uid), int64(gid), nil -} - -// Fill all of the fields of a Descriptor -func fillDescriptor(fimg *FileImage, index int, input DescriptorInput) (err error) { - descr := &fimg.DescrArr[index] - - curoff, err := fimg.Fp.Seek(0, 1) - if err != nil { - return fmt.Errorf("while file pointer look at: %s", err) - } - - descr.Datatype = input.Datatype - descr.ID = uint32(index) + 1 - descr.Used = true - descr.Groupid = input.Groupid - descr.Link = input.Link - align := os.Getpagesize() - if input.Alignment != 0 { - align = input.Alignment - } - descr.Fileoff, err = setFileOffNA(fimg, align) - if err != nil { - return - } - descr.Filelen = input.Size - descr.Storelen = descr.Fileoff + descr.Filelen - curoff - descr.Ctime = time.Now().Unix() - descr.Mtime = time.Now().Unix() - descr.UID, descr.Gid, err = getUserIDs() - if err != nil { - return fmt.Errorf("filling descriptor: %s", err) - } - descr.SetName(path.Base(input.Fname)) - descr.SetExtra(input.Extra.Bytes()) - - // Check that none or only 1 primary partition is ever set - if descr.Datatype == DataPartition { - ptype, err := descr.GetPartType() - if err != nil { - return err - } - if ptype == PartPrimSys { - if fimg.PrimPartID != 0 { - return fmt.Errorf("only 1 FS data object may be a primary partition") - } - fimg.PrimPartID = descr.ID - arch, err := descr.GetArch() - if err != nil { - return err - } - copy(fimg.Header.Arch[:], arch[:]) - } - } - - return -} - -// Write new data object to the SIF file -func writeDataObject(fimg *FileImage, index int, input DescriptorInput) error { - // if we have bytes in input.data use that instead of an input file - if input.Data != nil { - if _, err := fimg.Fp.Write(input.Data); err != nil { - return fmt.Errorf("copying data object data to SIF file: %s", err) - } - } else { - if n, err := io.Copy(fimg.Fp, input.Fp); err != nil { - return fmt.Errorf("copying data object file to SIF file: %s", err) - } else if n != input.Size && input.Size != 0 { - return fmt.Errorf("short write while copying to SIF file") - } else if input.Size == 0 { - // coming in from os.Stdin (pipe) - descr := &fimg.DescrArr[index] - descr.Filelen = n - descr.SetName("pipe" + fmt.Sprint(index+1)) - } - } - - return nil -} - -// Find a free descriptor and create a memory representation for addition to the SIF file -func createDescriptor(fimg *FileImage, input DescriptorInput) (err error) { - var ( - idx int - v Descriptor - ) - - if fimg.Header.Dfree == 0 { - return fmt.Errorf("no descriptor table free entry") - } - - // look for a free entry in the descriptor table - for idx, v = range fimg.DescrArr { - if !v.Used { - break - } - } - if int64(idx) == fimg.Header.Dtotal-1 && fimg.DescrArr[idx].Used { - return fmt.Errorf("no descriptor table free entry, warning: header.Dfree was > 0") - } - - // fill in SIF file descriptor - if err = fillDescriptor(fimg, idx, input); err != nil { - return - } - - // write data object associated to the descriptor in SIF file - if err = writeDataObject(fimg, idx, input); err != nil { - return fmt.Errorf("writing data object for SIF file: %s", err) - } - - // update some global header fields from adding this new descriptor - fimg.Header.Dfree-- - fimg.Header.Datalen += fimg.DescrArr[idx].Storelen - - return -} - -// Release and write the data object descriptor to backing storage (SIF container file) -func writeDescriptors(fimg *FileImage) error { - // first, move to descriptor start offset - if _, err := fimg.Fp.Seek(DescrStartOffset, 0); err != nil { - return fmt.Errorf("seeking to descriptor start offset: %s", err) - } - - for _, v := range fimg.DescrArr { - if err := binary.Write(fimg.Fp, binary.LittleEndian, v); err != nil { - return fmt.Errorf("binary writing descrtable to buf: %s", err) - } - } - fimg.Header.Descrlen = int64(binary.Size(fimg.DescrArr)) - - return nil -} - -// Write the global header to file -func writeHeader(fimg *FileImage) error { - // first, move to descriptor start offset - if _, err := fimg.Fp.Seek(0, 0); err != nil { - return fmt.Errorf("seeking to beginning of the file: %s", err) - } - - if err := binary.Write(fimg.Fp, binary.LittleEndian, fimg.Header); err != nil { - return fmt.Errorf("binary writing header to buf: %s", err) - } - - return nil -} - -// CreateContainer is responsible for the creation of a new SIF container -// file. It takes the creation information specification as input -// and produces an output file as specified in the input data. -func CreateContainer(cinfo CreateInfo) (fimg *FileImage, err error) { - fimg = &FileImage{} - fimg.DescrArr = make([]Descriptor, DescrNumEntries) - - // Prepare a fresh global header - copy(fimg.Header.Launch[:], cinfo.Launchstr) - copy(fimg.Header.Magic[:], HdrMagic) - copy(fimg.Header.Version[:], cinfo.Sifversion) - copy(fimg.Header.Arch[:], HdrArchUnknown) - copy(fimg.Header.ID[:], cinfo.ID[:]) - fimg.Header.Ctime = time.Now().Unix() - fimg.Header.Mtime = time.Now().Unix() - fimg.Header.Dfree = DescrNumEntries - fimg.Header.Dtotal = DescrNumEntries - fimg.Header.Descroff = DescrStartOffset - fimg.Header.Dataoff = DataStartOffset - - // Create container file - fimg.Fp, err = os.OpenFile(cinfo.Pathname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) - if err != nil { - return nil, fmt.Errorf("container file creation failed: %s", err) - } - defer fimg.Fp.Close() - - // set file pointer to start of data section */ - if _, err = fimg.Fp.Seek(DataStartOffset, 0); err != nil { - return nil, fmt.Errorf("setting file offset pointer to DataStartOffset: %s", err) - } - - for _, v := range cinfo.InputDescr { - if err = createDescriptor(fimg, v); err != nil { - return - } - } - - // Write down the descriptor array - if err = writeDescriptors(fimg); err != nil { - return - } - - // Write down global header to file - if err = writeHeader(fimg); err != nil { - return - } - - return -} - -func zeroData(fimg *FileImage, descr *Descriptor) error { - // first, move to data object offset - if _, err := fimg.Fp.Seek(descr.Fileoff, 0); err != nil { - return fmt.Errorf("seeking to data object offset: %s", err) - } - - var zero [4096]byte - n := descr.Filelen - upbound := int64(4096) - for { - if n < 4096 { - upbound = n - } - - if _, err := fimg.Fp.Write(zero[:upbound]); err != nil { - return fmt.Errorf("writing 0's to data object") - } - n -= 4096 - if n <= 0 { - break - } - } - - return nil -} - -func resetDescriptor(fimg *FileImage, index int) error { - // If we remove the primary partition, set the global header Arch field to HdrArchUnknown - // to indicate that the SIF file doesn't include a primary partition and no dependency - // on any architecture exists. - _, idx, _ := fimg.GetPartPrimSys() - if idx == index { - fimg.PrimPartID = 0 - copy(fimg.Header.Arch[:], HdrArchUnknown) - } - - offset := fimg.Header.Descroff + int64(index)*int64(binary.Size(fimg.DescrArr[0])) - - // first, move to descriptor offset - if _, err := fimg.Fp.Seek(offset, 0); err != nil { - return fmt.Errorf("seeking to descriptor: %s", err) - } - - var emptyDesc Descriptor - if err := binary.Write(fimg.Fp, binary.LittleEndian, emptyDesc); err != nil { - return fmt.Errorf("binary writing empty descriptor: %s", err) - } - - return nil -} - -// AddObject add a new data object and its descriptor into the specified SIF file. -func (fimg *FileImage) AddObject(input DescriptorInput) error { - // set file pointer to the end of data section - if _, err := fimg.Fp.Seek(fimg.Header.Dataoff+fimg.Header.Datalen, 0); err != nil { - return fmt.Errorf("setting file offset pointer to DataStartOffset: %s", err) - } - - // create a new descriptor entry from input data - if err := createDescriptor(fimg, input); err != nil { - return err - } - - // write down the descriptor array - if err := writeDescriptors(fimg); err != nil { - return err - } - - fimg.Header.Mtime = time.Now().Unix() - // write down global header to file - if err := writeHeader(fimg); err != nil { - return err - } - - if err := fimg.Fp.Sync(); err != nil { - return fmt.Errorf("while sync'ing new data object to SIF file: %s", err) - } - - return nil -} - -// descrIsLast return true if passed descriptor's object is the last in a SIF file -func objectIsLast(fimg *FileImage, descr *Descriptor) bool { - return fimg.Filesize == descr.Fileoff+descr.Filelen -} - -// compactAtDescr joins data objects leading and following "descr" by compacting a SIF file -func compactAtDescr(fimg *FileImage, descr *Descriptor) error { - var prev Descriptor - - for _, v := range fimg.DescrArr { - if !v.Used || v.ID == descr.ID { - continue - } else { - if v.Fileoff > prev.Fileoff { - prev = v - } - } - } - // make sure it's not the only used descriptor first - if prev.Used { - if err := fimg.Fp.Truncate(prev.Fileoff + prev.Filelen); err != nil { - return err - } - } else { - if err := fimg.Fp.Truncate(descr.Fileoff); err != nil { - return err - } - } - fimg.Header.Datalen -= descr.Storelen - return nil -} - -// DeleteObject removes data from a SIF file referred to by id. The descriptor for the -// data object is free'd and can be reused later. There's currenly 2 clean mode specified -// by flags: DelZero, to zero out the data region for security and DelCompact to -// remove and shink the file compacting the unused area. -func (fimg *FileImage) DeleteObject(id uint32, flags int) error { - descr, index, err := fimg.GetFromDescrID(id) - if err != nil { - return err - } - - switch flags { - case DelZero: - if err = zeroData(fimg, descr); err != nil { - return err - } - case DelCompact: - if objectIsLast(fimg, descr) { - if err = compactAtDescr(fimg, descr); err != nil { - return err - } - } else { - return fmt.Errorf("method (DelCompact) not implemented yet") - } - default: - if objectIsLast(fimg, descr) { - if err = compactAtDescr(fimg, descr); err != nil { - return err - } - } - } - - // update some global header fields from deleting this descriptor - fimg.Header.Dfree++ - fimg.Header.Mtime = time.Now().Unix() - - // zero out the unused descriptor - if err = resetDescriptor(fimg, index); err != nil { - return err - } - - // update global header - if err = writeHeader(fimg); err != nil { - return err - } - - if err := fimg.Fp.Sync(); err != nil { - return fmt.Errorf("while sync'ing deleted data object to SIF file: %s", err) - } - - return nil -} - -// SetPartExtra serializes the partition and fs type info into a binary buffer -func (di *DescriptorInput) SetPartExtra(fs Fstype, part Parttype, arch string) error { - extra := Partition{ - Fstype: fs, - Parttype: part, - } - if arch == HdrArchUnknown { - return fmt.Errorf("architecture not supported: %v", arch) - } - copy(extra.Arch[:], arch[:]) - - // serialize the partition data for integration with the base descriptor input - if err := binary.Write(&di.Extra, binary.LittleEndian, extra); err != nil { - return err - } - return nil -} - -// SetSignExtra serializes the hash type and the entity info into a binary buffer -func (di *DescriptorInput) SetSignExtra(hash Hashtype, entity string) error { - extra := Signature{ - Hashtype: hash, - } - - h, err := hex.DecodeString(entity) - if err != nil { - return err - } - copy(extra.Entity[:], h) - - // serialize the signature data for integration with the base descriptor input - if err := binary.Write(&di.Extra, binary.LittleEndian, extra); err != nil { - return err - } - return nil -} - -// SetName sets the byte array field "Name" to the value of string "name" -func (d *Descriptor) SetName(name string) { - copy(d.Name[:], []byte(name)) -} - -// SetExtra sets the extra byte array to a provided byte array -func (d *Descriptor) SetExtra(extra []byte) { - copy(d.Extra[:], extra) -} - -// SetPrimPart sets the specified system partition to be the primary one -func (fimg *FileImage) SetPrimPart(id uint32) error { - descr, _, err := fimg.GetFromDescrID(id) - if err != nil { - return err - } - - if descr.Datatype != DataPartition { - return fmt.Errorf("not a volume partition") - } - - ptype, err := descr.GetPartType() - if err != nil { - return err - } - - // if already primary system partition, nothing to do - if ptype == PartPrimSys { - return nil - } - - if ptype != PartSystem { - return fmt.Errorf("partition must be of system type") - } - - olddescr, _, err := fimg.GetPartPrimSys() - if err != nil && err != ErrNotFound { - return err - } - - fs, err := descr.GetFsType() - if err != nil { - return nil - } - - arch, err := descr.GetArch() - if err != nil { - return err - } - - copy(fimg.Header.Arch[:], arch[:]) - fimg.PrimPartID = descr.ID - - extra := Partition{ - Fstype: fs, - Parttype: PartPrimSys, - } - copy(extra.Arch[:], arch[:]) - - var extrabuf bytes.Buffer - if err := binary.Write(&extrabuf, binary.LittleEndian, extra); err != nil { - return err - } - descr.SetExtra(extrabuf.Bytes()) - - if olddescr != nil { - oldfs, err := olddescr.GetFsType() - if err != nil { - return nil - } - oldarch, err := olddescr.GetArch() - if err != nil { - return nil - } - - oldextra := Partition{ - Fstype: oldfs, - Parttype: PartSystem, - } - copy(oldextra.Arch[:], oldarch[:]) - - var oldextrabuf bytes.Buffer - if err := binary.Write(&oldextrabuf, binary.LittleEndian, oldextra); err != nil { - return err - } - olddescr.SetExtra(oldextrabuf.Bytes()) - } - - // write down the descriptor array - if err := writeDescriptors(fimg); err != nil { - return err - } - - fimg.Header.Mtime = time.Now().Unix() - // write down global header to file - if err := writeHeader(fimg); err != nil { - return err - } - - if err := fimg.Fp.Sync(); err != nil { - return fmt.Errorf("while sync'ing new data object to SIF file: %s", err) - } - - return nil -} diff --git a/sif/sif/init.go b/sif/sif/init.go deleted file mode 100644 index c5c84504c7..0000000000 --- a/sif/sif/init.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. -// Copyright (c) 2017, Yannick Cote All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -// +build linux - -package sif - -import ( - "bytes" - "log" -) - -var ( - sifLoggerBuf bytes.Buffer - siflog = log.New(&sifLoggerBuf, "", log.Ldate|log.Ltime|log.Lshortfile) -) - -func init() { -} diff --git a/sif/sif/load.go b/sif/sif/load.go deleted file mode 100644 index 610f5af66f..0000000000 --- a/sif/sif/load.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. -// Copyright (c) 2017, Yannick Cote All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -// +build linux - -package sif - -import ( - "bytes" - "encoding/binary" - "fmt" - "os" - "syscall" -) - -// Read the global header from the container file -func readHeader(fimg *FileImage) error { - if err := binary.Read(fimg.Reader, binary.LittleEndian, &fimg.Header); err != nil { - return fmt.Errorf("reading global header from container file: %s", err) - } - - return nil -} - -// Read the used descriptors and populate an in-memory representation of those in node list -func readDescriptors(fimg *FileImage) error { - // start by positioning us to the start of descriptors - _, err := fimg.Reader.Seek(fimg.Header.Descroff, 0) - if err != nil { - return fmt.Errorf("seek() setting to descriptors start: %s", err) - } - - // Initialize descriptor array (slice) and read them all from file - fimg.DescrArr = make([]Descriptor, fimg.Header.Dtotal) - if err := binary.Read(fimg.Reader, binary.LittleEndian, &fimg.DescrArr); err != nil { - fimg.DescrArr = nil - return fmt.Errorf("reading descriptor array from container file: %s", err) - } - - descr, _, err := fimg.GetPartPrimSys() - if err == nil { - fimg.PrimPartID = descr.ID - } - - return nil -} - -// Look at key fields from the global header to assess SIF validity. -// `runnable' checks is current container can run on host. -func isValidSif(fimg *FileImage) error { - // check various header fields - if string(fimg.Header.Magic[:HdrMagicLen-1]) != HdrMagic { - return fmt.Errorf("invalid SIF file: Magic |%s| want |%s|", fimg.Header.Magic, HdrMagic) - } - if string(fimg.Header.Version[:HdrVersionLen-1]) > HdrVersion { - return fmt.Errorf("invalid SIF file: Version %s want <= %s", fimg.Header.Version, HdrVersion) - } - - return nil -} - -// mapFile takes a file pointer and returns a slice of bytes representing the file data -func (fimg *FileImage) mapFile(rdonly bool) error { - prot := syscall.PROT_READ - flags := syscall.MAP_PRIVATE - - info, err := fimg.Fp.Stat() - if err != nil { - return fmt.Errorf("while trying to size SIF file to mmap") - } - fimg.Filesize = info.Size() - - size := nextAligned(info.Size(), syscall.Getpagesize()) - if int64(int(size)) < info.Size() { - return fmt.Errorf("file is to big to be mapped") - } - - if !rdonly { - prot = syscall.PROT_WRITE - flags = syscall.MAP_SHARED - } - - fimg.Filedata, err = syscall.Mmap(int(fimg.Fp.Fd()), 0, int(size), prot, flags) - if err != nil { - // mmap failed, use sequential read() instead for top of file - siflog.Printf("mmap on %s failed, reading buffer sequentially...", fimg.Fp.Name()) - fimg.Filedata = make([]byte, DataStartOffset) - - // start by positioning us to the start of the file - _, err := fimg.Fp.Seek(0, 0) - if err != nil { - return fmt.Errorf("seek() setting to start of file: %s", err) - } - - if n, err := fimg.Fp.Read(fimg.Filedata); n != DataStartOffset { - return fmt.Errorf("short read while reading top of file: %v", err) - - } - fimg.Amodebuf = true - } - - // create and associate a new bytes.Reader on top of mmap'ed or buffered data from file - fimg.Reader = bytes.NewReader(fimg.Filedata) - - return nil -} - -func (fimg *FileImage) unmapFile() error { - if fimg.Amodebuf { - return nil - } - if err := syscall.Munmap(fimg.Filedata); err != nil { - return fmt.Errorf("while calling unmapping SIF file") - } - return nil -} - -// LoadContainer is responsible for loading a SIF container file. It takes -// the container file name, and whether the file is opened as read-only -// as arguments. -func LoadContainer(filename string, rdonly bool) (fimg FileImage, err error) { - if rdonly { // open SIF rdonly if mounting immutable partitions or inspecting the image - if fimg.Fp, err = os.Open(filename); err != nil { - return fimg, fmt.Errorf("opening(RDONLY) container file: %s", err) - } - } else { // open SIF read-write when adding and removing data objects - if fimg.Fp, err = os.OpenFile(filename, os.O_RDWR, 0644); err != nil { - return fimg, fmt.Errorf("opening(RDWR) container file: %s", err) - } - } - - // get a memory map of the SIF file - if err = fimg.mapFile(rdonly); err != nil { - return - } - - // read global header from SIF file - if err = readHeader(&fimg); err != nil { - return - } - - // validate global header - if err = isValidSif(&fimg); err != nil { - return - } - - // read descriptor array from SIF file - if err = readDescriptors(&fimg); err != nil { - return - } - - return -} - -// LoadContainerFp is responsible for loading a SIF container file. It takes -// a *os.File pointing to an opened file, and whether the file is opened as -// read-only for arguments. -func LoadContainerFp(fp *os.File, rdonly bool) (fimg FileImage, err error) { - if fp == nil { - return fimg, fmt.Errorf("provided fp for file is invalid") - } - - fimg.Fp = fp - - // get a memory map of the SIF file - if err = fimg.mapFile(rdonly); err != nil { - return - } - - // read global header from SIF file - if err = readHeader(&fimg); err != nil { - return - } - - // validate global header - if err = isValidSif(&fimg); err != nil { - return - } - - // read descriptor array from SIF file - if err = readDescriptors(&fimg); err != nil { - return - } - - return fimg, nil -} - -// LoadContainerReader is responsible for processing SIF data from a byte stream -// and extract various components like the global header, descriptors and even -// perhaps data, depending on how much is read from the source. -func LoadContainerReader(b *bytes.Reader) (fimg FileImage, err error) { - fimg.Reader = b - - // read global header from SIF file - if err = readHeader(&fimg); err != nil { - return - } - - // validate global header - if err = isValidSif(&fimg); err != nil { - return - } - - // in the case where the reader buffer doesn't include descriptor data, we - // don't return an error and DescrArr will be set to nil - _ = readDescriptors(&fimg) - - return fimg, nil -} - -// UnloadContainer closes the SIF container file and free associated resources if needed -func (fimg *FileImage) UnloadContainer() (err error) { - // if SIF data comes from file, not a slice buffer (see LoadContainer() variants) - if fimg.Fp != nil { - if err = fimg.unmapFile(); err != nil { - return - } - if err = fimg.Fp.Close(); err != nil { - return fmt.Errorf("closing SIF file failed, corrupted: don't use: %s", err) - } - } - return -} diff --git a/sif/sif/lookup.go b/sif/sif/lookup.go deleted file mode 100644 index 363050827b..0000000000 --- a/sif/sif/lookup.go +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. -// Copyright (c) 2017, Yannick Cote All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -// +build linux - -package sif - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "strings" -) - -// ErrNotFound is the code for when no search key is not found -var ErrNotFound = errors.New("no match found") - -// ErrMultValues is the code for when search key is not unique -var ErrMultValues = errors.New("lookup would return more than one match") - -// -// Methods on (fimg *FIleImage) -// - -// GetSIFArch returns the SIF arch code from go runtime arch code -func GetSIFArch(goarch string) (sifarch string) { - var ok bool - - archMap := map[string]string{ - "386": HdrArch386, - "amd64": HdrArchAMD64, - "arm": HdrArchARM, - "arm64": HdrArchARM64, - "ppc64": HdrArchPPC64, - "ppc64le": HdrArchPPC64le, - "mips": HdrArchMIPS, - "mipsle": HdrArchMIPSle, - "mips64": HdrArchMIPS64, - "mips64le": HdrArchMIPS64le, - "s390x": HdrArchS390x, - } - - if sifarch, ok = archMap[goarch]; !ok { - sifarch = HdrArchUnknown - } - return sifarch -} - -// GetGoArch returns the go runtime arch code from the SIF arch code -func GetGoArch(sifarch string) (goarch string) { - var ok bool - - archMap := map[string]string{ - HdrArch386: "386", - HdrArchAMD64: "amd64", - HdrArchARM: "arm", - HdrArchARM64: "arm64", - HdrArchPPC64: "ppc64", - HdrArchPPC64le: "ppc64le", - HdrArchMIPS: "mips", - HdrArchMIPSle: "mipsle", - HdrArchMIPS64: "mips64", - HdrArchMIPS64le: "mips64le", - HdrArchS390x: "s390x", - } - - if goarch, ok = archMap[sifarch]; !ok { - goarch = "unknown" - } - return goarch -} - -// GetHeader returns the loaded SIF global header -func (fimg *FileImage) GetHeader() *Header { - return &fimg.Header -} - -// GetFromDescrID searches for a descriptor with -func (fimg *FileImage) GetFromDescrID(id uint32) (*Descriptor, int, error) { - var match = -1 - - for i, v := range fimg.DescrArr { - if !v.Used { - continue - } else { - if v.ID == id { - if match != -1 { - return nil, -1, ErrMultValues - } - match = i - } - } - } - - if match == -1 { - return nil, -1, ErrNotFound - } - - return &fimg.DescrArr[match], match, nil -} - -// GetPartFromGroup searches for partition descriptors inside a specific group -func (fimg *FileImage) GetPartFromGroup(groupid uint32) ([]*Descriptor, []int, error) { - var descrs []*Descriptor - var indexes []int - var count int - - for i, v := range fimg.DescrArr { - if !v.Used { - continue - } else { - if v.Datatype == DataPartition && v.Groupid == groupid { - indexes = append(indexes, i) - descrs = append(descrs, &fimg.DescrArr[i]) - count++ - } - } - } - - if count == 0 { - return nil, nil, ErrNotFound - } - - return descrs, indexes, nil -} - -// GetSignFromGroup searches for signature descriptors inside a specific group -func (fimg *FileImage) GetSignFromGroup(groupid uint32) ([]*Descriptor, []int, error) { - var descrs []*Descriptor - var indexes []int - var count int - - for i, v := range fimg.DescrArr { - if !v.Used { - continue - } else { - if v.Datatype == DataSignature && v.Groupid == groupid { - indexes = append(indexes, i) - descrs = append(descrs, &fimg.DescrArr[i]) - count++ - } - } - } - - if count == 0 { - return nil, nil, ErrNotFound - } - - return descrs, indexes, nil -} - -// GetFromLinkedDescr searches for descriptors that point to "id" -func (fimg *FileImage) GetFromLinkedDescr(ID uint32) ([]*Descriptor, []int, error) { - var descrs []*Descriptor - var indexes []int - var count int - - for i, v := range fimg.DescrArr { - if !v.Used { - continue - } else { - if v.Link == ID { - indexes = append(indexes, i) - descrs = append(descrs, &fimg.DescrArr[i]) - count++ - } - } - } - - if count == 0 { - return nil, nil, ErrNotFound - } - - return descrs, indexes, nil -} - -// GetFromDescr searches for descriptors comparing all non-nil fields of a provided descriptor -func (fimg *FileImage) GetFromDescr(descr Descriptor) ([]*Descriptor, []int, error) { - var descrs []*Descriptor - var indexes []int - var count int - - for i, v := range fimg.DescrArr { - if !v.Used { - continue - } else { - if descr.Datatype != 0 && descr.Datatype != v.Datatype { - continue - } - if descr.ID != 0 && descr.ID != v.ID { - continue - } - if descr.Groupid != 0 && descr.Groupid != v.Groupid { - continue - } - if descr.Link != 0 && descr.Link != v.Link { - continue - } - if descr.Fileoff != 0 && descr.Fileoff != v.Fileoff { - continue - } - if descr.Filelen != 0 && descr.Filelen != v.Filelen { - continue - } - if descr.Storelen != 0 && descr.Storelen != v.Storelen { - continue - } - if descr.Ctime != 0 && descr.Ctime != v.Ctime { - continue - } - if descr.Mtime != 0 && descr.Mtime != v.Mtime { - continue - } - if descr.UID != 0 && descr.UID != v.UID { - continue - } - if descr.Gid != 0 && descr.Gid != v.Gid { - continue - } - if descr.Name[0] != 0 && !bytes.Equal(descr.Name[:], v.Name[:]) { - continue - } - - indexes = append(indexes, i) - descrs = append(descrs, &fimg.DescrArr[i]) - count++ - } - } - - if count == 0 { - return nil, nil, ErrNotFound - } - - return descrs, indexes, nil -} - -// -// Methods on (descr *Descriptor) -// - -// GetData return a memory mapped byte slice mirroring the data object in a SIF file. -func (descr *Descriptor) GetData(fimg *FileImage) []byte { - if fimg.Amodebuf { - _, err := fimg.Fp.Seek(descr.Fileoff, 0) - if err != nil { - return nil - } - data := make([]byte, descr.Filelen) - if n, _ := fimg.Fp.Read(data); int64(n) != descr.Filelen { - return nil - } - return data - } - - return fimg.Filedata[descr.Fileoff : descr.Fileoff+descr.Filelen] -} - -// GetName returns the name tag associated with the descriptor. Analogous to file name. -func (descr *Descriptor) GetName() string { - return strings.TrimRight(string(descr.Name[:]), "\000") -} - -// GetFsType extracts the Fstype field from the Extra field of a Partition Descriptor -func (descr *Descriptor) GetFsType() (Fstype, error) { - if descr.Datatype != DataPartition { - return -1, fmt.Errorf("expected DataPartition, got %v", descr.Datatype) - } - - var pinfo Partition - b := bytes.NewReader(descr.Extra[:]) - if err := binary.Read(b, binary.LittleEndian, &pinfo); err != nil { - return -1, fmt.Errorf("while extracting Partition extra info: %s", err) - } - - return pinfo.Fstype, nil -} - -// GetPartType extracts the Parttype field from the Extra field of a Partition Descriptor -func (descr *Descriptor) GetPartType() (Parttype, error) { - if descr.Datatype != DataPartition { - return -1, fmt.Errorf("expected DataPartition, got %v", descr.Datatype) - } - - var pinfo Partition - b := bytes.NewReader(descr.Extra[:]) - if err := binary.Read(b, binary.LittleEndian, &pinfo); err != nil { - return -1, fmt.Errorf("while extracting Partition extra info: %s", err) - } - - return pinfo.Parttype, nil -} - -// GetArch extracts the Arch field from the Extra field of a Partition Descriptor -func (descr *Descriptor) GetArch() ([HdrArchLen]byte, error) { - if descr.Datatype != DataPartition { - return [HdrArchLen]byte{}, fmt.Errorf("expected DataPartition, got %v", descr.Datatype) - } - - var pinfo Partition - b := bytes.NewReader(descr.Extra[:]) - if err := binary.Read(b, binary.LittleEndian, &pinfo); err != nil { - return [HdrArchLen]byte{}, fmt.Errorf("while extracting Partition extra info: %s", err) - } - - return pinfo.Arch, nil -} - -// GetHashType extracts the Hashtype field from the Extra field of a Signature Descriptor -func (descr *Descriptor) GetHashType() (Hashtype, error) { - if descr.Datatype != DataSignature { - return -1, fmt.Errorf("expected DataSignature, got %v", descr.Datatype) - } - - var sinfo Signature - b := bytes.NewReader(descr.Extra[:]) - if err := binary.Read(b, binary.LittleEndian, &sinfo); err != nil { - return -1, fmt.Errorf("while extracting Signature extra info: %s", err) - } - - return sinfo.Hashtype, nil -} - -// GetEntity extracts the signing entity field from the Extra field of a Signature Descriptor -func (descr *Descriptor) GetEntity() ([]byte, error) { - if descr.Datatype != DataSignature { - return nil, fmt.Errorf("expected DataSignature, got %v", descr.Datatype) - } - - var sinfo Signature - b := bytes.NewReader(descr.Extra[:]) - if err := binary.Read(b, binary.LittleEndian, &sinfo); err != nil { - return nil, fmt.Errorf("while extracting Signature extra info: %s", err) - } - - return sinfo.Entity[:], nil -} - -// GetEntityString returns the string version of the stored entity -func (descr *Descriptor) GetEntityString() (string, error) { - fingerprint, err := descr.GetEntity() - if err != nil { - return "", err - } - - return fmt.Sprintf("%0X", fingerprint[:20]), nil -} - -// GetPartPrimSys returns the primary system partition if present. There should -// be only one primary system partition in a SIF file. -func (fimg *FileImage) GetPartPrimSys() (*Descriptor, int, error) { - var descr *Descriptor - index := -1 - - for i, v := range fimg.DescrArr { - if !v.Used { - continue - } else { - if v.Datatype == DataPartition { - ptype, err := v.GetPartType() - if err != nil { - return nil, -1, err - } - if ptype == PartPrimSys { - if index != -1 { - return nil, -1, ErrMultValues - } - index = i - descr = &fimg.DescrArr[i] - } - } - } - } - - if index == -1 { - return nil, -1, ErrNotFound - } - - return descr, index, nil -} diff --git a/sif/sif/sif.go b/sif/sif/sif.go deleted file mode 100644 index 964d7b72f7..0000000000 --- a/sif/sif/sif.go +++ /dev/null @@ -1,297 +0,0 @@ -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. -// Copyright (c) 2017, Yannick Cote All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -// +build linux - -// Package sif implements data structures and routines to create -// and access SIF files. -// - sif.go contains the data definition the file format. -// - create.go implements the core functionality for the creation of -// of new SIF files. -// - load.go implements the core functionality for the loading of -// existing SIF files. -// - lookup.go mostly implements search/lookup and printing routines -// and access to specific descriptor/data found in SIF container files. -package sif - -import ( - "bytes" - "os" - - uuid "github.com/satori/go.uuid" -) - -// Layout of a SIF file (example) -// -// .================================================. -// | GLOBAL HEADER: Sifheader | -// | - launch: "#!/usr/bin/env..." | -// | - magic: "SIF_MAGIC" | -// | - version: "1" | -// | - arch: "4" | -// | - uuid: b2659d4e-bd50-4ea5-bd17-eec5e54f918e | -// | - ctime: 1504657553 | -// | - mtime: 1504657653 | -// | - ndescr: 3 | -// | - descroff: 120 | --. -// | - descrlen: 432 | | -// | - dataoff: 4096 | | -// | - datalen: 619362 | | -// |------------------------------------------------| <-' -// | DESCR[0]: Sifdeffile | -// | - Sifcommon | -// | - datatype: DATA_DEFFILE | -// | - id: 1 | -// | - groupid: 1 | -// | - link: NONE | -// | - fileoff: 4096 | --. -// | - filelen: 222 | | -// |------------------------------------------------| <-----. -// | DESCR[1]: Sifpartition | | | -// | - Sifcommon | | | -// | - datatype: DATA_PARTITION | | | -// | - id: 2 | | | -// | - groupid: 1 | | | -// | - link: NONE | | | -// | - fileoff: 4318 | ----. | -// | - filelen: 618496 | | | | -// | - fstype: Squashfs | | | | -// | - parttype: System | | | | -// | - content: Linux | | | | -// |------------------------------------------------| | | | -// | DESCR[2]: Sifsignature | | | | -// | - Sifcommon | | | | -// | - datatype: DATA_SIGNATURE | | | | -// | - id: 3 | | | | -// | - groupid: NONE | | | | -// | - link: 2 | ------' -// | - fileoff: 622814 | ------. -// | - filelen: 644 | | | | -// | - hashtype: SHA384 | | | | -// | - entity: @ | | | | -// |------------------------------------------------| <-' | | -// | Definition file data | | | -// | . | | | -// | . | | | -// | . | | | -// |------------------------------------------------| <---' | -// | File system partition image | | -// | . | | -// | . | | -// | . | | -// |------------------------------------------------| <-----' -// | Signed verification data | -// | . | -// | . | -// | . | -// `================================================' - -// SIF header constants and quantities -const ( - HdrLaunch = "#!/usr/bin/env run-singularity\n" - HdrMagic = "SIF_MAGIC" // SIF identification - HdrVersion = "02" // SIF SPEC VERSION - HdrArchUnknown = "00" // Undefined/Unsupported arch - HdrArch386 = "01" // 386 (i[3-6]86) arch code - HdrArchAMD64 = "02" // AMD64 arch code - HdrArchARM = "03" // ARM arch code - HdrArchARM64 = "04" // AARCH64 arch code - HdrArchPPC64 = "05" // PowerPC 64 arch code - HdrArchPPC64le = "06" // PowerPC 64 little-endian arch code - HdrArchMIPS = "07" // MIPS arch code - HdrArchMIPSle = "08" // MIPS little-endian arch code - HdrArchMIPS64 = "09" // MIPS64 arch code - HdrArchMIPS64le = "10" // MIPS64 little-endian arch code - HdrArchS390x = "11" // IBM s390x arch code - - HdrLaunchLen = 32 // len("#!/usr/bin/env... ") - HdrMagicLen = 10 // len("SIF_MAGIC") - HdrVersionLen = 3 // len("99") - HdrArchLen = 3 // len("99") - - DescrNumEntries = 48 // the default total number of available descriptors - DescrGroupMask = 0xf0000000 // groups start at that offset - DescrUnusedGroup = DescrGroupMask // descriptor without a group - DescrDefaultGroup = DescrGroupMask | 1 // first groupid number created - DescrUnusedLink = 0 // descriptor without link to other - DescrEntityLen = 256 // len("Joe Bloe ...") - DescrNameLen = 128 // descriptor name (string identifier) - DescrMaxPrivLen = 384 // size reserved for descriptor specific data - DescrStartOffset = 4096 // where descriptors start after global header - DataStartOffset = 32768 // where data object start after descriptors -) - -// Datatype represents the different SIF data object types stored in the image -type Datatype int32 - -// List of supported SIF data types -const ( - DataDeffile Datatype = iota + 0x4001 // definition file data object - DataEnvVar // environment variables data object - DataLabels // JSON labels data object - DataPartition // file system data object - DataSignature // signing/verification data object - DataGenericJSON // generic JSON meta-data - DataGeneric // generic / raw data -) - -// Fstype represents the different SIF file system types found in partition data objects -type Fstype int32 - -// List of supported file systems -const ( - FsSquash Fstype = iota + 1 // Squashfs file system, RDONLY - FsExt3 // EXT3 file system, RDWR (deprecated) - FsImmuObj // immutable data object archive - FsRaw // raw data -) - -// Parttype represents the different SIF container partition types (system and data) -type Parttype int32 - -// List of supported partition types -const ( - PartSystem Parttype = iota + 1 // partition hosts an operating system - PartPrimSys // partition hosts the primary operating system - PartData // partition hosts data only - PartOverlay // partition hosts an overlay -) - -// Hashtype represents the different SIF hashing function types used to fingerprint data objects -type Hashtype int32 - -// List of supported hash functions -const ( - HashSHA256 Hashtype = iota + 1 - HashSHA384 - HashSHA512 - HashBLAKE2S - HashBLAKE2B -) - -// SIF data object deletation strategies -const ( - DelZero = iota + 1 // zero the data object bytes - DelCompact // free the space used by data object -) - -// Descriptor represents the SIF descriptor type -type Descriptor struct { - Datatype Datatype // informs of descriptor type - Used bool // is the descriptor in use - ID uint32 // a unique id for this data object - Groupid uint32 // object group this data object is related to - Link uint32 // special link or relation to an id or group - Fileoff int64 // offset from start of image file - Filelen int64 // length of data in file - Storelen int64 // length of data + alignment to store data in file - - Ctime int64 // image creation time - Mtime int64 // last modification time - UID int64 // system user owning the file - Gid int64 // system group owning the file - Name [DescrNameLen]byte // descriptor name (string identifier) - Extra [DescrMaxPrivLen]byte // big enough for extra data below -} - -// Deffile represents the SIF definition-file data object descriptor -type Deffile struct { -} - -// Labels represents the SIF JSON-labels data object descriptor -type Labels struct { -} - -// Envvar represents the SIF envvar data object descriptor -type Envvar struct { -} - -// Partition represents the SIF partition data object descriptor -type Partition struct { - Fstype Fstype - Parttype Parttype - Arch [HdrArchLen]byte // arch the image is built for -} - -// Signature represents the SIF signature data object descriptor -type Signature struct { - Hashtype Hashtype - Entity [DescrEntityLen]byte -} - -// GenericJSON represents the SIF generic JSON meta-data data object descriptor -type GenericJSON struct { -} - -// Generic represents the SIF generic data object descriptor -type Generic struct { -} - -// Header describes a loaded SIF file -type Header struct { - Launch [HdrLaunchLen]byte // #! shell execution line - - Magic [HdrMagicLen]byte // look for "SIF_MAGIC" - Version [HdrVersionLen]byte // SIF version - Arch [HdrArchLen]byte // arch the primary partition is built for - ID uuid.UUID // image unique identifier - - Ctime int64 // image creation time - Mtime int64 // last modification time - - Dfree int64 // # of unused data object descr. - Dtotal int64 // # of total available data object descr. - Descroff int64 // bytes into file where descs start - Descrlen int64 // bytes used by all current descriptors - Dataoff int64 // bytes into file where data starts - Datalen int64 // bytes used by all data objects -} - -// -// This section describes SIF creation/loading data structures used when -// building or opening a SIF file. Transient data not found in the final -// SIF file. Those data structures are internal. -// - -// FileImage describes the representation of a SIF file in memory -type FileImage struct { - Header Header // the loaded SIF global header - Fp *os.File // file pointer of opened SIF file - Filesize int64 // file size of the opened SIF file - Filedata []byte // the content of the opened file - Amodebuf bool // access mode: mmap = false, buffered = true - Reader *bytes.Reader // reader on top of Mapdata - DescrArr []Descriptor // slice of loaded descriptors from SIF file - PrimPartID uint32 // ID of primary system partition if present -} - -// CreateInfo wraps all SIF file creation info needed -type CreateInfo struct { - Pathname string // the end result output filename - Launchstr string // the shell run command - Sifversion string // the SIF specification version used - ID uuid.UUID // image unique identifier - InputDescr []DescriptorInput // slice of input info for descriptor creation -} - -// DescriptorInput describes the common info needed to create a data object descriptor -type DescriptorInput struct { - Datatype Datatype // datatype being harvested for new descriptor - Groupid uint32 // group to be set for new descriptor - Link uint32 // link to be set for new descriptor - Size int64 // size of the data object for the new descriptor - Alignment int // Align requirement for data object - - Fname string // file containing data associated with the new descriptor - Fp *os.File // file pointer to opened 'fname' - Data []byte // loaded data from file - - Image *FileImage // loaded SIF file in memory - Descr *Descriptor // created end result descriptor - - Extra bytes.Buffer // where specific input type store their data -} From 44ef87df65bdcde499284225861fc1e1a1b2c31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Thu, 6 Jan 2022 22:01:42 +0100 Subject: [PATCH 06/11] Re-update golang.org/x/crypto MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Undo the version downgrades in commit "sif: use upstream sif module". Signed-off-by: Miloslav Trmač --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2ea23d2f86..1393701fc9 100644 --- a/go.mod +++ b/go.mod @@ -38,8 +38,8 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect github.com/xeipuuv/gojsonschema v1.2.0 go.etcd.io/bbolt v1.3.6 - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b + golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 diff --git a/go.sum b/go.sum index 0809f6ebe1..f89d9af1f6 100644 --- a/go.sum +++ b/go.sum @@ -854,8 +854,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -941,8 +942,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b h1:SXy8Ld8oKlcogOvUAh0J5Pm5RKzgYBMMxLxt6n5XW50= -golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From 67c18a6477e2003aeb0684341d0f1fee191fead1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Thu, 6 Jan 2022 23:41:04 +0100 Subject: [PATCH 07/11] Update build directives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > gofmt -s -w . fixes build. Signed-off-by: Miloslav Trmač --- sif/sif_src.go | 1 + sif/sif_transport.go | 1 + transports/alltransports/sif.go | 1 + transports/alltransports/sif_stub.go | 1 + 4 files changed, 4 insertions(+) diff --git a/sif/sif_src.go b/sif/sif_src.go index 92af266472..79295cbc47 100644 --- a/sif/sif_src.go +++ b/sif/sif_src.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package sifimage diff --git a/sif/sif_transport.go b/sif/sif_transport.go index 17814aa64a..841b538137 100644 --- a/sif/sif_transport.go +++ b/sif/sif_transport.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package sifimage diff --git a/transports/alltransports/sif.go b/transports/alltransports/sif.go index ba348f2d16..687f2ff77f 100644 --- a/transports/alltransports/sif.go +++ b/transports/alltransports/sif.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package alltransports diff --git a/transports/alltransports/sif_stub.go b/transports/alltransports/sif_stub.go index 7715f7be0b..8567427b4a 100644 --- a/transports/alltransports/sif_stub.go +++ b/transports/alltransports/sif_stub.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package alltransports From 04e1b8d4b6430e11fc161b17efeeb969c2b398f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Wed, 12 Jan 2022 23:35:51 +0100 Subject: [PATCH 08/11] Rename sif/* files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename sif/sif_* to remove the sif_ prefix The directory name is sufficient disambiguation, so decrease the visual noise and repetition. - Remove the sif/internal subpackage We don't need an extra subpackage, just move the code to the only user. This makes the internal code publicly-visible, we'll change that immediately. Signed-off-by: Miloslav Trmač --- sif/{internal/sif_util.go => load.go} | 2 +- sif/{sif_src.go => src.go} | 5 ++--- sif/{sif_transport.go => transport.go} | 0 3 files changed, 3 insertions(+), 4 deletions(-) rename sif/{internal/sif_util.go => load.go} (99%) rename sif/{sif_src.go => src.go} (98%) rename sif/{sif_transport.go => transport.go} (100%) diff --git a/sif/internal/sif_util.go b/sif/load.go similarity index 99% rename from sif/internal/sif_util.go rename to sif/load.go index ed3eabf948..cec80cff05 100644 --- a/sif/internal/sif_util.go +++ b/sif/load.go @@ -1,7 +1,7 @@ //go:build linux // +build linux -package internal +package sifimage import ( "bufio" diff --git a/sif/sif_src.go b/sif/src.go similarity index 98% rename from sif/sif_src.go rename to sif/src.go index 79295cbc47..8389b8c1f6 100644 --- a/sif/sif_src.go +++ b/sif/src.go @@ -14,7 +14,6 @@ import ( "time" "github.com/containers/image/v5/internal/tmpdir" - "github.com/containers/image/v5/sif/internal" "github.com/containers/image/v5/types" "github.com/klauspost/pgzip" "github.com/opencontainers/go-digest" @@ -26,7 +25,7 @@ import ( type sifImageSource struct { ref sifReference - sifimg internal.SifImage + sifimg SifImage workdir string diffID digest.Digest diffSize int64 @@ -108,7 +107,7 @@ func (s *sifImageSource) getBlobInfo() error { func newImageSource(ctx context.Context, sys *types.SystemContext, ref sifReference) (types.ImageSource, error) { var imgSrc sifImageSource - sifimg, err := internal.LoadSIFImage(ref.resolvedFile) + sifimg, err := LoadSIFImage(ref.resolvedFile) if err != nil { return nil, errors.Wrap(err, "loading SIF file") } diff --git a/sif/sif_transport.go b/sif/transport.go similarity index 100% rename from sif/sif_transport.go rename to sif/transport.go From eb8254ba7bc3715f36c02712e317ac3e614fbc98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Fri, 7 Jan 2022 00:44:14 +0100 Subject: [PATCH 09/11] Make the sif/load.go code package-private MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ... after moving it from a sif/internal subpackage, make it private again. Signed-off-by: Miloslav Trmač --- sif/load.go | 28 ++++++++++++++-------------- sif/src.go | 4 ++-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/sif/load.go b/sif/load.go index cec80cff05..84b8cdaa54 100644 --- a/sif/load.go +++ b/sif/load.go @@ -20,7 +20,7 @@ import ( imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) -type SifImage struct { +type loadedSifImage struct { fimg *sif.FileImage rootfs sif.Descriptor deffile *sif.Descriptor @@ -32,7 +32,7 @@ type SifImage struct { envlist []string } -func LoadSIFImage(path string) (image SifImage, err error) { +func loadSIFImage(path string) (image loadedSifImage, err error) { // open up the SIF file and get its header image.fimg, err = sif.LoadContainerFromPath(path, sif.OptLoadWithFlag(os.O_RDONLY)) if err != nil { @@ -42,7 +42,7 @@ func LoadSIFImage(path string) (image SifImage, err error) { // check for a system partition and save it image.rootfs, err = image.fimg.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys)) if err != nil { - return SifImage{}, errors.Wrap(err, "looking up rootfs from SIF file") + return loadedSifImage{}, errors.Wrap(err, "looking up rootfs from SIF file") } // look for a definition file object @@ -53,7 +53,7 @@ func LoadSIFImage(path string) (image SifImage, err error) { image.defReader = resultDesc.GetReader() } if err = image.generateConfig(); err != nil { - return SifImage{}, err + return loadedSifImage{}, err } // look for an environment variable set object @@ -67,7 +67,7 @@ func LoadSIFImage(path string) (image SifImage, err error) { return image, nil } -func (image *SifImage) parseEnvironment(scanner *bufio.Scanner) error { +func (image *loadedSifImage) parseEnvironment(scanner *bufio.Scanner) error { for scanner.Scan() { s := strings.TrimSpace(scanner.Text()) if s == "" || strings.HasPrefix(s, "#") { @@ -84,7 +84,7 @@ func (image *SifImage) parseEnvironment(scanner *bufio.Scanner) error { return nil } -func (image *SifImage) parseRunscript(scanner *bufio.Scanner) error { +func (image *loadedSifImage) parseRunscript(scanner *bufio.Scanner) error { for scanner.Scan() { s := strings.TrimSpace(scanner.Text()) if strings.HasPrefix(s, "%") { @@ -98,7 +98,7 @@ func (image *SifImage) parseRunscript(scanner *bufio.Scanner) error { return nil } -func (image *SifImage) generateRunscript() error { +func (image *loadedSifImage) generateRunscript() error { base := `#!/bin/bash ` image.runscript = bytes.NewBufferString(base) @@ -117,7 +117,7 @@ func (image *SifImage) generateRunscript() error { return nil } -func (image *SifImage) generateConfig() error { +func (image *loadedSifImage) generateConfig() error { if image.deffile == nil { image.cmdlist = append(image.cmdlist, "bash") return nil @@ -159,21 +159,21 @@ func (image *SifImage) generateConfig() error { return nil } -func (image SifImage) GetConfig(config *imgspecv1.Image) error { +func (image loadedSifImage) GetConfig(config *imgspecv1.Image) error { config.Config.Cmd = append(config.Config.Cmd, image.cmdlist...) return nil } -func (image SifImage) UnloadSIFImage() (err error) { +func (image loadedSifImage) UnloadSIFImage() (err error) { err = image.fimg.UnloadContainer() return } -func (image SifImage) GetSIFID() string { +func (image loadedSifImage) GetSIFID() string { return image.fimg.ID() } -func (image SifImage) GetSIFArch() string { +func (image loadedSifImage) GetSIFArch() string { return image.fimg.PrimaryArch() } @@ -198,7 +198,7 @@ unsquashfs -f ` + squashFilename + ` && tar --acls --xattrs -C ./squashfs-root - return } -func (image *SifImage) writeRunscript(tempdir string) (err error) { +func (image *loadedSifImage) writeRunscript(tempdir string) (err error) { if image.runscript == nil { return nil } @@ -212,7 +212,7 @@ func (image *SifImage) writeRunscript(tempdir string) (err error) { return nil } -func (image SifImage) SquashFSToTarLayer(tempdir string) (tarpath string, err error) { +func (image loadedSifImage) SquashFSToTarLayer(tempdir string) (tarpath string, err error) { f, err := os.Create(filepath.Join(tempdir, squashFilename)) if err != nil { return diff --git a/sif/src.go b/sif/src.go index 8389b8c1f6..1623a54994 100644 --- a/sif/src.go +++ b/sif/src.go @@ -25,7 +25,7 @@ import ( type sifImageSource struct { ref sifReference - sifimg SifImage + sifimg loadedSifImage workdir string diffID digest.Digest diffSize int64 @@ -107,7 +107,7 @@ func (s *sifImageSource) getBlobInfo() error { func newImageSource(ctx context.Context, sys *types.SystemContext, ref sifReference) (types.ImageSource, error) { var imgSrc sifImageSource - sifimg, err := LoadSIFImage(ref.resolvedFile) + sifimg, err := loadSIFImage(ref.resolvedFile) if err != nil { return nil, errors.Wrap(err, "loading SIF file") } From 71dfda12c94ef9919c9875860f9a7a88a2bae823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Wed, 5 Jan 2022 21:11:14 +0100 Subject: [PATCH 10/11] Allow building the SIF transport on non-Linux systems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It probabaly doesn't _work_ right now, at least macOS is missing a working fakeroot. We do intend to avoid the use of fakeroot eventually. (Adventurous experimenting developers might provide a no-op "fakeroot" script on most platforms.) Also, having the code compile on macOS significantly helps development. Signed-off-by: Miloslav Trmač --- sif/load.go | 3 --- sif/src.go | 3 --- sif/transport.go | 3 --- transports/alltransports/alltransports.go | 2 +- transports/alltransports/sif.go | 9 --------- transports/alltransports/sif_stub.go | 10 ---------- 6 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 transports/alltransports/sif.go delete mode 100644 transports/alltransports/sif_stub.go diff --git a/sif/load.go b/sif/load.go index 84b8cdaa54..12cd89b571 100644 --- a/sif/load.go +++ b/sif/load.go @@ -1,6 +1,3 @@ -//go:build linux -// +build linux - package sifimage import ( diff --git a/sif/src.go b/sif/src.go index 1623a54994..d7d4251ff2 100644 --- a/sif/src.go +++ b/sif/src.go @@ -1,6 +1,3 @@ -//go:build linux -// +build linux - package sifimage import ( diff --git a/sif/transport.go b/sif/transport.go index 841b538137..3b53f85c0f 100644 --- a/sif/transport.go +++ b/sif/transport.go @@ -1,6 +1,3 @@ -//go:build linux -// +build linux - package sifimage import ( diff --git a/transports/alltransports/alltransports.go b/transports/alltransports/alltransports.go index 9dc9b4c28b..0bae8b2599 100644 --- a/transports/alltransports/alltransports.go +++ b/transports/alltransports/alltransports.go @@ -12,9 +12,9 @@ import ( _ "github.com/containers/image/v5/oci/archive" _ "github.com/containers/image/v5/oci/layout" _ "github.com/containers/image/v5/openshift" + _ "github.com/containers/image/v5/sif" _ "github.com/containers/image/v5/tarball" - // The sif transport is registered by sif*.go // The ostree transport is registered by ostree*.go // The storage transport is registered by storage*.go "github.com/containers/image/v5/transports" diff --git a/transports/alltransports/sif.go b/transports/alltransports/sif.go deleted file mode 100644 index 687f2ff77f..0000000000 --- a/transports/alltransports/sif.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build linux -// +build linux - -package alltransports - -import ( - // Register the sif transport - _ "github.com/containers/image/v5/sif" -) diff --git a/transports/alltransports/sif_stub.go b/transports/alltransports/sif_stub.go deleted file mode 100644 index 8567427b4a..0000000000 --- a/transports/alltransports/sif_stub.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !linux -// +build !linux - -package alltransports - -import "github.com/containers/image/v5/transports" - -func init() { - transports.Register(transports.NewStubTransport("sif")) -} From a281e36b0aa42fc98defa646d9f20c6fc5b2ec22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Fri, 7 Jan 2022 00:27:36 +0100 Subject: [PATCH 11/11] Extensive refactoring to address review comments and hopefully simplify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix policy configuration identities in sif - Actually allow something in ValidatePolicyConfigurationScope ; SIF is one of the cases where it's actually a bit plausible that a policy rejecting some filesystem sources might be desirable. - Fix PolicyConfigurationNamespaces not to include the file name itself, and "/" Add tests for sifTransport and sifReference The NewImage and NewImageSource tests are rather pointless, but we don't want to require and invoke fakeroot etc. on every unit test run, at least for now. Use ref.file instead of ref.resolvedFile in newImageSource Consistently with the dir: design, if the user specifies a relative path, use it directly so that we don't introduce races against changes to the directory structure. Beautify sif_transport.go - Use the usual order (transport implementation, followed by reference implementation) - Copy&paste more of the comments, to reinforce the contract requirements. Fix the package name directive Reorganize imports ... to follow the usual convention. Don't use pkg/errors in sif. Mostly replace its uses with fmt.Errorf(...%w...). Fix uses of fmt.Errorf - Use %w instead of %v for error wrapping - Use errors.New when the string is constant Don't prefix wrapped error context with "error " ... to match most of c/image code, where we have previously removed such prefixes. Return true from HasThreadSafeGetBlob ... because that's the case for the current implementation, although it makes no difference for the current c/image/copy caller, when this source only provides one layer. Rename sifImageSource.blobID to blobDigest Rename sifImageSource.configID to configDigest Remove workdir if newImageSource fails Rename sifImageSource.blob* to layer* ... to differentiate the layer data from the config, which is also a "blob" in the ImageSource naming. Remove sifImageSource.diffID The value only needs to be known inside newImageSource, so pass it around in a variable/return value. Remove unused sifImageSource.diffSize ... which allows us to make getLayerInfo a function without a (partially-created) sifImageSource parent object. Move layerTime computation from createBlob to getBlobInfo If anything, the latter is a bit more accurate (capturing the time of the last update of the file we are creating, vs. the time of the initial creation), but we want to eventually that with a value from the SIF header anyway. Remove sifImageSource.layerTime It's only necessary in newImageSource, so use a return value and a local variable for that. Remove unused sifImageSource.layerType Remove sifImageSource.configSize This value is already stored in the sifImageSource.config slice, so don't store a redundant copy. Rename workdir to workDir following usual Go patterns. Rename tarpath to tarPath everywhere Return layerDigest and layerSize from getBlobInfo ... instead of writing it to partially-initialized sifImageSource. Provide a path to getBlobInfo instead of reading it from sifImageSource This makes getBlobInfo independent of the partially-created sifImageSource. Don't create a compressed layer from the SIF file Compression is very costly, principially in CPU time. Many use cases (notably import to c/storage) would only end up decompressing the data again. Those that do neeed the data compressed, like push to a registry, can use the copy pipeline's streaming compression implementation, often without needing to store the compressed version in a temporary file. So this is likely to improve both CPU time usage and (maximum) disk space usage - at the very least against the current implementation which doens't even remove the uncompressed version after creating the compressed one :) This is a minimal version of the change, we are now computing the layer's digest twice. We'll fix that soon. Rename tarPath to layerPath ... to be consistent with the other variables. Don't compute the DiffID separately It's the same value as the layer digest, now that the layer is just the uncompressed tarball. Rename fgz to f The file is now expected to not be compressed. Rename blobDigester to digester ... just to be a bit shorter. Inline some single-use variables when building the manifest Use a struct initializer instead of a set of assignments for config Inline single-use variables when building a config Also remove some fairly redundant comments. Use a switch if sifImageSource.GetBlob ... to make the structure a tiny bit less repetitive. Close the SIF image object in newImageSource Nothing actually needs it afterwards. Rename UnloadSIFImage() to Close() to indirectly silence a linter about handling the error; it can't fail in practice, and isn't quite worth handling. Explicitly specify a MediaType field in the generated OCI manifest ... to follow best practices vs. schema confusion attacks (although this generated manfiest is clearly not a schema confusion attack). Beautify loadedSifImage.Close Turn loadedSifImage.GetConfig into CommandLine - Make loadedSifImage independent of the OCI format details. - Make it clear at the call site that only the command is actually provided. - Don't return an error value which is always nil, which makes the caller simpler. Remove lookup of the sif.DataEnvVar descriptor It is unused in this codebase, and it's unclear what, if anything, it is used for anywhere else. Make SifImage.parseEnvironment and SifImage.parseRunscript stand-alone ... i.e. independent from SifImage, so that we can more easily unit-test it. The res *[]string parameter is rather ugly, but we'll refactor it away soon enough. Should not change behavior. Split parseDefFile from SifImage.generateConfig ... to have an easily unit-testable bit of code. Should not change behavior. This destructively assigns to image.envlist and image.cmdlist instead of appending, but it should be the only writer at that point. Add a smoke test for parseDefFile Use a state machine for parseDefFile ... instead of a nesting scanner.Scan() loops and a goto. Should not change behavior. Remove a misleading comment Now, with GetDescriptor, more than one matching descriptor results in an error, so there isn't anything to assume about a single value. Move DataDeffile descriptor lookup into generateConfig It's the only user of that data Remove deffile and defReader from loadedSifImage Turn them into trivial local variables in generateConfig() The code remains a big convoluted, we'll clean that up soon. Simplify generateConfig Eliminate both deffile and defReader. Pass %environment and %runscript to generateRunscript explicitly That will eventually make it easier to unit-test Return the generated script from generateRunscript instad of updating image This makes generateRunscript stand-alone and easy to unit-test. Add a smoke test for generateRunscript It's not much, but better than nothing. Use InjectedScript instead of Runscript for the script we generate ... everywhere, to differentiate that script from the %runscript section contents. Store injectedScript as a []byte instead of bytes.Buffer No need to keep around the intermediate form, and this allows us to change the implementation. Use strings.Join and Sprintf instead of bytes.Buffer in generateInjectedScript Assuming this is not performance-critical, the code is much shorter, and clearly cannot fail (just like the previous version, which is documented to panic rather than return the errors that version unnecessarily handled). Note that this might change behavior for empty %environment or %runscript sections: we now add extra empty lines. That shouldn't make a difference. Remove the unnecessary error return value from generateInjectedScript Remove loadedSifImage.envlist All users are local to generateConfig. Don't use loadedSifImage.cmdlist for storing %runscript It's just a local value to generateConfig, and we no longer use cmdlist for both %runscript and the final command line. Replace loadedSifImage.cmdlist with a single command string The array only ever has one element, so get rid of the array. Beautify generateConfig Always refer to environment and runscript in the same order. Move generateInjectedScript after parseDefFile First create the data, then consume it. Simplify generateConfig Only have the fallback to "bash" if no script is available in a single place. Rename resultDesc to desc For such a short-lived variable we can have a shorter name. Rename tempdir to tempDir throughout ... to follow Go conventions. Remove a Sync call on the squashfs copy We'd actually prefer that data not to hit the disk; we want to remove it as soon as possible. Instead, scope the deferred Close() so that it happens before we consume the file. Pass around squashFSPath in a variable. ... instead of providing an ambient constant for the relative path. That will make it clearer which code uses that file. Pass around tarPath in a variable. ... instead of providing an ambient constant for the relative path. That will make it clearer which code uses that file. Remove the generated tar file on failure e.g. if creating it runs out of space. Beautify SquashFSToTarLayer Explicitly return values instead of relying on named return values to make the data (in this case, error) flow more explicit. Use Sprintf instead of string concatenation for the generated script It is a bit more manageable that way. Also actually start the script with a recognized shebang instead of a newline. Don't hard-code "squashfs-root" all over the place. Pass extractedRootPath around instead, and use (unsquashfs -d) to override the built-in default. Inline a single-use cmd variable Rename xcmd to cmd ... now that the cmd name is available. Use explicit return statements instead of named return values Make loadedImage always passed by reference The struct contains a stateful *sif.FileImage, which makes no sense to copy; so don't get into that habit even in cases where it might be safe. Use a constant for the /podman/runscript path ... instead of hard-coding it over the place, and even assuming a specific directory structure. Add more context to write failures Don't write to stderr; return error output to the caller Remove the generated script immediately after using it Make the tar file creation cancellable using the provided context Also add TODO notes in other places where we would prefer the copy to be cancellable. Rename runUnSquashFSTar to createTarFromSIFInputs We are going to have it handle the injectedScript as well. Pass scriptPath to exec.Command instead of hard-coding a constant This allows us not to care about the working directory of the script, as well. Move the scriptPath decision to SquashFSToTarLayer That's the only place that is aware of tempDir now. Pass injectedScript to writeInjectedScript ... to make it independent of loadedSifImage; it will go away entirely soon. Call writeInjectedScript from createTarFromSIFInputs ... so that createTarFromSIFInputs is responsible for both creating and consuming extractedRootPath, without any external interference. Move the cleanup of extractedRootPath to createTarFromSIFInputs ... to make it a tiny bit more self-contained, now that it handles the injectedScript as well. Add a comment to more clearly document the alocation of paths in SquashFSToTarLayer Remove loadedSifImage.rootfs Instead, determine the value in SquashFSToTarLayer. This means that we now try to interpret the deffile before checking for rootfs presence, changing the possible order of errors. That shouldn't be much of a difference for valid images. Rename generateConfig to processDefFile Have processDefFile return the values instead of writing to loadedSifImage We will eliminate the loadedSifImage members entirely soon. Pass sif.FileImage to processDefFile explicitly ... instead of using the image.fimg member. Rename SquashFSToTarLayer to convertSIFToElements We are going to have it return other values as well. Return also the command line from convertSIFToElements This turns it into the central point of the conversion process, instead of the fairly ambient loadedSifImage object. Call processDefFile only in convertSIFToElements This allows us to remove loadedSifImage.injectedScript and loadedSifImage.command, making loadedSifImage finally a trivial wrapper around sif.FileImage - and we'll eliminate that wrapper next. Inline loadedSifImage.GetSIFID We don't really need that abstraction. Inline loadedSifImage.GetSIFArch We don't really need that abstraction. Make convertSIFToElements a stand-alone function Eliminating the last non-trivial user of loadedSifImage. Eliminate loadedSifImage Finally, eliminate the loadedSifImage type entirely. It doesn't really make sense to inject a layer of abstraction between sifImageSource and sif.FileImage, purely for the abstraction. loadedSifImage was only ever used in one way, as an essentially procedural step; that is now served by the convertSIFToElements function, rather than being split between the loadedSifImage constructor and the original tarball creation method. (convertSIFToElements might eventually return a struct with named fields if there were many, but it doesn't make sense for newImageSource to hold an object and fill it up one step at a time.) Use the last modification time from the SIF header for OCI creation time Add a TODO note Add a note about (unsquashfs -o) Add debug logs around long-running operations ... and make sure to include paths of the relevant files. Signed-off-by: Miloslav Trmač --- sif/load.go | 336 ++++++++++++++++++++---------------------- sif/load_test.go | 58 ++++++++ sif/src.go | 269 +++++++++++++-------------------- sif/transport.go | 87 ++++++++--- sif/transport_test.go | 177 ++++++++++++++++++++++ 5 files changed, 560 insertions(+), 367 deletions(-) create mode 100644 sif/load_test.go create mode 100644 sif/transport_test.go diff --git a/sif/load.go b/sif/load.go index 12cd89b571..2dbeb7da9b 100644 --- a/sif/load.go +++ b/sif/load.go @@ -1,8 +1,8 @@ -package sifimage +package sif import ( "bufio" - "bytes" + "context" "fmt" "io" "io/ioutil" @@ -11,221 +11,201 @@ import ( "path/filepath" "strings" - "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/sylabs/sif/v2/pkg/sif" - - imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) -type loadedSifImage struct { - fimg *sif.FileImage - rootfs sif.Descriptor - deffile *sif.Descriptor - defReader io.Reader - cmdlist []string - runscript *bytes.Buffer - env *sif.Descriptor - envReader io.Reader - envlist []string -} - -func loadSIFImage(path string) (image loadedSifImage, err error) { - // open up the SIF file and get its header - image.fimg, err = sif.LoadContainerFromPath(path, sif.OptLoadWithFlag(os.O_RDONLY)) - if err != nil { - return - } +// injectedScriptTargetPath is the path injectedScript should be written to in the created image. +const injectedScriptTargetPath = "/podman/runscript" - // check for a system partition and save it - image.rootfs, err = image.fimg.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys)) - if err != nil { - return loadedSifImage{}, errors.Wrap(err, "looking up rootfs from SIF file") - } +// parseDefFile parses a SIF definition file from reader, +// and returns non-trivial contents of the %environment and %runscript sections. +func parseDefFile(reader io.Reader) ([]string, []string, error) { + type parserState int + const ( + parsingOther parserState = iota + parsingEnvironment + parsingRunscript + ) - // look for a definition file object - resultDesc, err := image.fimg.GetDescriptor(sif.WithDataType(sif.DataDeffile)) - if err == nil { - // we assume in practice that typical SIF files don't hold multiple deffiles - image.deffile = &resultDesc - image.defReader = resultDesc.GetReader() - } - if err = image.generateConfig(); err != nil { - return loadedSifImage{}, err - } + environment := []string{} + runscript := []string{} - // look for an environment variable set object - resultDesc, err = image.fimg.GetDescriptor(sif.WithDataType(sif.DataEnvVar)) - if err == nil { - // we assume in practice that typical SIF files don't hold multiple EnvVar sets - image.env = &resultDesc - image.envReader = resultDesc.GetReader() - } - - return image, nil -} - -func (image *loadedSifImage) parseEnvironment(scanner *bufio.Scanner) error { + state := parsingOther + scanner := bufio.NewScanner(reader) for scanner.Scan() { s := strings.TrimSpace(scanner.Text()) - if s == "" || strings.HasPrefix(s, "#") { - continue - } - if strings.HasPrefix(s, "%") { - return nil + switch { + case s == `%environment`: + state = parsingEnvironment + case s == `%runscript`: + state = parsingRunscript + case strings.HasPrefix(s, "%"): + state = parsingOther + case state == parsingEnvironment: + if s != "" && !strings.HasPrefix(s, "#") { + environment = append(environment, s) + } + case state == parsingRunscript: + runscript = append(runscript, s) + default: // parsingOther: ignore the line } - image.envlist = append(image.envlist, s) } if err := scanner.Err(); err != nil { - return errors.Wrap(err, "parsing environment from SIF definition file object") + return nil, nil, fmt.Errorf("reading lines from SIF definition file object: %w", err) } - return nil + return environment, runscript, nil } -func (image *loadedSifImage) parseRunscript(scanner *bufio.Scanner) error { - for scanner.Scan() { - s := strings.TrimSpace(scanner.Text()) - if strings.HasPrefix(s, "%") { - return nil - } - image.cmdlist = append(image.cmdlist, s) - } - if err := scanner.Err(); err != nil { - return errors.Wrap(err, "parsing runscript from SIF definition file object") - } - return nil +// generateInjectedScript generates a shell script based on +// SIF definition file %environment and %runscript data, and returns it. +func generateInjectedScript(environment []string, runscript []string) []byte { + script := fmt.Sprintf("#!/bin/bash\n"+ + "%s\n"+ + "%s\n", strings.Join(environment, "\n"), strings.Join(runscript, "\n")) + return []byte(script) } -func (image *loadedSifImage) generateRunscript() error { - base := `#!/bin/bash -` - image.runscript = bytes.NewBufferString(base) - for _, s := range image.envlist { - _, err := image.runscript.WriteString(fmt.Sprintln(s)) +// processDefFile finds sif.DataDeffile in sifImage, if any, +// and returns: +// - the command to run +// - contents of a script to inject as injectedScriptTargetPath, or nil +func processDefFile(sifImage *sif.FileImage) (string, []byte, error) { + var environment, runscript []string + + desc, err := sifImage.GetDescriptor(sif.WithDataType(sif.DataDeffile)) + if err == nil { + environment, runscript, err = parseDefFile(desc.GetReader()) if err != nil { - return errors.Wrap(err, "writing to runscript buffer") + return "", nil, err } } - for _, s := range image.cmdlist { - _, err := image.runscript.WriteString(fmt.Sprintln(s)) - if err != nil { - return errors.Wrap(err, "writing to runscript buffer") - } + + var command string + var injectedScript []byte + if len(environment) == 0 && len(runscript) == 0 { + command = "bash" + injectedScript = nil + } else { + injectedScript = generateInjectedScript(environment, runscript) + command = injectedScriptTargetPath } - return nil + + return command, injectedScript, nil } -func (image *loadedSifImage) generateConfig() error { - if image.deffile == nil { - image.cmdlist = append(image.cmdlist, "bash") +func writeInjectedScript(extractedRootPath string, injectedScript []byte) error { + if injectedScript == nil { return nil } - - // extract %environment/%runscript from definition file - var err error - scanner := bufio.NewScanner(image.defReader) - for scanner.Scan() { - s := strings.TrimSpace(scanner.Text()) - again: - if s == `%environment` { - if err = image.parseEnvironment(scanner); err != nil { - return err - } - } else if s == `%runscript` { - if err = image.parseRunscript(scanner); err != nil { - return err - } - } - s = strings.TrimSpace(scanner.Text()) - if s == `%environment` || s == `%runscript` { - goto again - } + filePath := filepath.Join(extractedRootPath, injectedScriptTargetPath) + parentDirPath := filepath.Dir(filePath) + if err := os.MkdirAll(parentDirPath, 0755); err != nil { + return fmt.Errorf("creating %s: %w", parentDirPath, err) } - if err := scanner.Err(); err != nil { - return errors.Wrap(err, "reading lines from SIF definition file object") - } - - if len(image.cmdlist) == 0 && len(image.envlist) == 0 { - image.cmdlist = append(image.cmdlist, "bash") - } else { - if err = image.generateRunscript(); err != nil { - return errors.Wrap(err, "generating runscript") - } - image.cmdlist = []string{"/podman/runscript"} + if err := ioutil.WriteFile(filePath, injectedScript, 0755); err != nil { + return fmt.Errorf("writing %s to %s: %w", injectedScriptTargetPath, filePath, err) } - return nil } -func (image loadedSifImage) GetConfig(config *imgspecv1.Image) error { - config.Config.Cmd = append(config.Config.Cmd, image.cmdlist...) - return nil -} - -func (image loadedSifImage) UnloadSIFImage() (err error) { - err = image.fimg.UnloadContainer() - return -} - -func (image loadedSifImage) GetSIFID() string { - return image.fimg.ID() -} - -func (image loadedSifImage) GetSIFArch() string { - return image.fimg.PrimaryArch() -} - -const squashFilename = "rootfs.squashfs" -const tarFilename = "rootfs.tar" - -func runUnSquashFSTar(tempdir string) (err error) { - script := ` -#!/bin/sh -unsquashfs -f ` + squashFilename + ` && tar --acls --xattrs -C ./squashfs-root -cpf ` + tarFilename + ` ./ -` - - if err = ioutil.WriteFile(filepath.Join(tempdir, "script"), []byte(script), 0755); err != nil { +// createTarFromSIFInputs creates a tar file at tarPath, using a squashfs image at squashFSPath. +// It can also use extractedRootPath and scriptPath, which are allocated for its exclusive use, +// if necessary. +func createTarFromSIFInputs(ctx context.Context, tarPath, squashFSPath string, injectedScript []byte, extractedRootPath, scriptPath string) error { + // It's safe for the Remove calls to happen even before we create the files, because tempDir is exclusive + // for our use. + defer os.RemoveAll(extractedRootPath) + + // Almost everything in extractedRootPath comes from squashFSPath. + conversionCommand := fmt.Sprintf("unsquashfs -d %s -f %s && tar --acls --xattrs -C %s -cpf %s ./", + extractedRootPath, squashFSPath, extractedRootPath, tarPath) + script := "#!/bin/sh\n" + conversionCommand + "\n" + if err := ioutil.WriteFile(scriptPath, []byte(script), 0755); err != nil { return err } - cmd := []string{"fakeroot", "--", "./script"} + defer os.Remove(scriptPath) - xcmd := exec.Command(cmd[0], cmd[1:]...) - xcmd.Stderr = os.Stderr - xcmd.Dir = tempdir - err = xcmd.Run() - return -} - -func (image *loadedSifImage) writeRunscript(tempdir string) (err error) { - if image.runscript == nil { - return nil - } - rsPath := filepath.Join(tempdir, "squashfs-root", "podman") - if err = os.MkdirAll(rsPath, 0755); err != nil { - return + // On top of squashFSPath, we only add injectedScript, if necessary. + if err := writeInjectedScript(extractedRootPath, injectedScript); err != nil { + return err } - if err = ioutil.WriteFile(filepath.Join(rsPath, "runscript"), image.runscript.Bytes(), 0755); err != nil { - return errors.Wrap(err, "writing /podman/runscript") + + logrus.Debugf("Converting squashfs to tar, command: %s ...", conversionCommand) + cmd := exec.CommandContext(ctx, "fakeroot", "--", scriptPath) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("converting image: %w, output: %s", err, string(output)) } + logrus.Debugf("... finished converting squashfs to tar") return nil } -func (image loadedSifImage) SquashFSToTarLayer(tempdir string) (tarpath string, err error) { - f, err := os.Create(filepath.Join(tempdir, squashFilename)) +// convertSIFToElements processes sifImage and creates/returns +// the relevant elements for contructing an OCI-like image: +// - A path to a tar file containing a root filesystem, +// - A command to run. +// The returned tar file path is inside tempDir, which can be assumed to be empty +// at start, and is exclusively used by the current process (i.e. it is safe +// to use hard-coded relative paths within it). +func convertSIFToElements(ctx context.Context, sifImage *sif.FileImage, tempDir string) (string, []string, error) { + // We could allocate unique names for all of these using ioutil.Temp*, but tempDir is exclusive, + // so we can just hard-code a set of unique values here. + // We create and/or manage cleanup of these two paths. + squashFSPath := filepath.Join(tempDir, "rootfs.squashfs") + tarPath := filepath.Join(tempDir, "rootfs.tar") + // We only allocate these paths, the user is responsible for cleaning them up. + extractedRootPath := filepath.Join(tempDir, "rootfs") + scriptPath := filepath.Join(tempDir, "script") + + succeeded := false + // It's safe for the Remove calls to happen even before we create the files, because tempDir is exclusive + // for our use. + // Ideally we would remove squashFSPath immediately after creating extractedRootPath, but we need + // to run both creation and consumption of extractedRootPath in the same fakeroot context. + // So, overall, this process requires at least 2 compressed copies (SIF and squashFSPath) and 2 + // uncompressed copies (extractedRootPath and tarPath) of the data, all using up space at the same time. + // That's rather unsatisfactory, ideally we would be streaming the data directly from a squashfs parser + // reading from the SIF file to a tarball, for 1 compresed and 1 uncompressed copy. + defer os.Remove(squashFSPath) + defer func() { + if !succeeded { + os.Remove(tarPath) + } + }() + + command, injectedScript, err := processDefFile(sifImage) if err != nil { - return + return "", nil, err } - defer f.Close() - if _, err = io.CopyN(f, image.rootfs.GetReader(), image.rootfs.Size()); err != nil { - return - } - if err = f.Sync(); err != nil { - return - } - if err = image.writeRunscript(tempdir); err != nil { - return + + rootFS, err := sifImage.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys)) + if err != nil { + return "", nil, fmt.Errorf("looking up rootfs from SIF file: %w", err) + } + // TODO: We'd prefer not to make a full copy of the file here; unsquashfs ≥ 4.4 + // has an -o option that allows extracting a squashfs from the SIF file directly, + // but that version is not currently available in RHEL 8. + logrus.Debugf("Creating a temporary squashfs image %s ...", squashFSPath) + if err := func() error { // A scope for defer + f, err := os.Create(squashFSPath) + if err != nil { + return err + } + defer f.Close() + // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done(). + if _, err := io.CopyN(f, rootFS.GetReader(), rootFS.Size()); err != nil { + return err + } + return nil + }(); err != nil { + return "", nil, err } - if err = runUnSquashFSTar(tempdir); err != nil { - return + logrus.Debugf("... finished creating a temporary squashfs image") + + if err := createTarFromSIFInputs(ctx, tarPath, squashFSPath, injectedScript, extractedRootPath, scriptPath); err != nil { + return "", nil, err } - return filepath.Join(tempdir, tarFilename), nil + succeeded = true + return tarPath, []string{command}, nil } diff --git a/sif/load_test.go b/sif/load_test.go new file mode 100644 index 0000000000..ee9bc7261e --- /dev/null +++ b/sif/load_test.go @@ -0,0 +1,58 @@ +package sif + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseDefFile(t *testing.T) { + for _, c := range []struct { + name string + input string + environment []string + runscript []string + }{ + {"Empty input", "", []string{}, []string{}}, + { + name: "Basic smoke test", + input: "Bootstrap: library\n" + + "%environment\n" + + " export FOO=world\n" + + " export BAR=baz\n" + + "%runscript\n" + + ` echo "Hello $FOO"` + "\n" + + " sleep 5\n" + + "%help\n" + + " Abandon all hope.\n", + environment: []string{"export FOO=world", "export BAR=baz"}, + runscript: []string{`echo "Hello $FOO"`, "sleep 5"}, + }, + { + name: "Trailing section marker", + input: "Bootstrap: library\n" + + "%environment\n" + + " export FOO=world\n" + + "%runscript", + environment: []string{"export FOO=world"}, + runscript: []string{}, + }, + } { + env, rs, err := parseDefFile(bytes.NewReader([]byte(c.input))) + require.NoError(t, err, c.name) + assert.Equal(t, c.environment, env, c.name) + assert.Equal(t, c.runscript, rs, c.name) + } +} + +func TestGenerateInjectedScript(t *testing.T) { + res := generateInjectedScript([]string{"export FOO=world", "export BAR=baz"}, + []string{`echo "Hello $FOO"`, "sleep 5"}) + assert.Equal(t, "#!/bin/bash\n"+ + "export FOO=world\n"+ + "export BAR=baz\n"+ + `echo "Hello $FOO"`+"\n"+ + "sleep 5\n", string(res)) +} diff --git a/sif/src.go b/sif/src.go index d7d4251ff2..ba95a469f3 100644 --- a/sif/src.go +++ b/sif/src.go @@ -1,217 +1,151 @@ -package sifimage +package sif import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "io/ioutil" "os" - "time" "github.com/containers/image/v5/internal/tmpdir" "github.com/containers/image/v5/types" - "github.com/klauspost/pgzip" "github.com/opencontainers/go-digest" - "github.com/pkg/errors" - imgspecs "github.com/opencontainers/image-spec/specs-go" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" + "github.com/sylabs/sif/v2/pkg/sif" ) type sifImageSource struct { - ref sifReference - sifimg loadedSifImage - workdir string - diffID digest.Digest - diffSize int64 - blobID digest.Digest - blobSize int64 - blobTime time.Time - blobType string - blobFile string - config []byte - configID digest.Digest - configSize int64 - manifest []byte -} - -func (s *sifImageSource) getLayerInfo(tarpath string) error { - ftar, err := os.Open(tarpath) - if err != nil { - return fmt.Errorf("error opening %q for reading: %v", tarpath, err) - } - defer ftar.Close() - - diffDigester := digest.Canonical.Digester() - s.diffSize, err = io.Copy(diffDigester.Hash(), ftar) - if err != nil { - return fmt.Errorf("error reading %q: %v", tarpath, err) - } - s.diffID = diffDigester.Digest() - - return nil -} -func (s *sifImageSource) createBlob(tarpath string) error { - s.blobFile = fmt.Sprintf("%s.%s", tarpath, "gz") - fgz, err := os.Create(s.blobFile) - if err != nil { - return errors.Wrapf(err, "creating file for compressed blob") - } - defer fgz.Close() - fileinfo, err := fgz.Stat() - if err != nil { - return fmt.Errorf("error reading modtime of %q: %v", s.blobFile, err) - } - s.blobTime = fileinfo.ModTime() - - ftar, err := os.Open(tarpath) - if err != nil { - return fmt.Errorf("error opening %q for reading: %v", tarpath, err) - } - defer ftar.Close() - - writer := pgzip.NewWriter(fgz) - defer writer.Close() - _, err = io.Copy(writer, ftar) - if err != nil { - return fmt.Errorf("error compressing %q: %v", tarpath, err) - } - - return nil + ref sifReference + workDir string + layerDigest digest.Digest + layerSize int64 + layerFile string + config []byte + configDigest digest.Digest + manifest []byte } -func (s *sifImageSource) getBlobInfo() error { - fgz, err := os.Open(s.blobFile) +// getBlobInfo returns the digest, and size of the provided file. +func getBlobInfo(path string) (digest.Digest, int64, error) { + f, err := os.Open(path) if err != nil { - return fmt.Errorf("error opening %q for reading: %v", s.blobFile, err) + return "", -1, fmt.Errorf("opening %q for reading: %w", path, err) } - defer fgz.Close() + defer f.Close() - blobDigester := digest.Canonical.Digester() - s.blobSize, err = io.Copy(blobDigester.Hash(), fgz) + // TODO: Instead of writing the tar file to disk, and reading + // it here again, stream the tar file to a pipe and + // compute the digest while writing it to disk. + logrus.Debugf("Computing a digest of the SIF conversion output...") + digester := digest.Canonical.Digester() + // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done(). + size, err := io.Copy(digester.Hash(), f) if err != nil { - return fmt.Errorf("error reading %q: %v", s.blobFile, err) + return "", -1, fmt.Errorf("reading %q: %w", path, err) } - s.blobID = blobDigester.Digest() + digest := digester.Digest() + logrus.Debugf("... finished computing the digest of the SIF conversion output") - return nil + return digest, size, nil } // newImageSource returns an ImageSource for reading from an existing directory. // newImageSource extracts SIF objects and saves them in a temp directory. func newImageSource(ctx context.Context, sys *types.SystemContext, ref sifReference) (types.ImageSource, error) { - var imgSrc sifImageSource - - sifimg, err := loadSIFImage(ref.resolvedFile) - if err != nil { - return nil, errors.Wrap(err, "loading SIF file") - } - - workdir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "sif") + sifImg, err := sif.LoadContainerFromPath(ref.file, sif.OptLoadWithFlag(os.O_RDONLY)) if err != nil { - return nil, errors.Wrapf(err, "creating temp directory") + return nil, fmt.Errorf("loading SIF file: %w", err) } + defer func() { + _ = sifImg.UnloadContainer() + }() - tarpath, err := sifimg.SquashFSToTarLayer(workdir) + workDir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "sif") if err != nil { - return nil, errors.Wrapf(err, "converting rootfs from SquashFS to Tarball") - } - - // generate layer info - err = imgSrc.getLayerInfo(tarpath) - if err != nil { - return nil, errors.Wrapf(err, "gathering layer diff information") + return nil, fmt.Errorf("creating temp directory: %w", err) } + succeeded := false + defer func() { + if !succeeded { + os.RemoveAll(workDir) + } + }() - // prepare compressed blob - err = imgSrc.createBlob(tarpath) + layerPath, commandLine, err := convertSIFToElements(ctx, sifImg, workDir) if err != nil { - return nil, errors.Wrapf(err, "creating blob file") + return nil, fmt.Errorf("converting rootfs from SquashFS to Tarball: %w", err) } - // generate blob info - err = imgSrc.getBlobInfo() + layerDigest, layerSize, err := getBlobInfo(layerPath) if err != nil { - return nil, errors.Wrapf(err, "gathering blob information") + return nil, fmt.Errorf("gathering blob information: %w", err) } - // populate the rootfs section of the config - rootfs := imgspecv1.RootFS{ - Type: "layers", - DiffIDs: []digest.Digest{imgSrc.diffID}, - } - created := imgSrc.blobTime - history := []imgspecv1.History{ - { - Created: &created, - CreatedBy: fmt.Sprintf("/bin/sh -c #(nop) ADD file:%s in %c", imgSrc.diffID.Hex(), os.PathSeparator), - Comment: "imported from SIF, uuid: " + sifimg.GetSIFID(), + created := sifImg.ModifiedAt() + config := imgspecv1.Image{ + Created: &created, + Architecture: sifImg.PrimaryArch(), + OS: "linux", + Config: imgspecv1.ImageConfig{ + Cmd: commandLine, }, - { - Created: &created, - CreatedBy: "/bin/sh -c #(nop) CMD [\"bash\"]", - EmptyLayer: true, + RootFS: imgspecv1.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{layerDigest}, + }, + History: []imgspecv1.History{ + { + Created: &created, + CreatedBy: fmt.Sprintf("/bin/sh -c #(nop) ADD file:%s in %c", layerDigest.Hex(), os.PathSeparator), + Comment: "imported from SIF, uuid: " + sifImg.ID(), + }, + { + Created: &created, + CreatedBy: "/bin/sh -c #(nop) CMD [\"bash\"]", + EmptyLayer: true, + }, }, } - - // build an OCI image config - var config imgspecv1.Image - config.Created = &created - config.Architecture = sifimg.GetSIFArch() - config.OS = "linux" - config.RootFS = rootfs - config.History = history - err = sifimg.GetConfig(&config) - if err != nil { - return nil, errors.Wrapf(err, "getting config elements from SIF") - } - - // Encode and digest the image configuration blob. configBytes, err := json.Marshal(&config) if err != nil { - return nil, fmt.Errorf("error generating configuration blob for %q: %v", ref.resolvedFile, err) + return nil, fmt.Errorf("generating configuration blob for %q: %w", ref.resolvedFile, err) } - configID := digest.Canonical.FromBytes(configBytes) - configSize := int64(len(configBytes)) + configDigest := digest.Canonical.FromBytes(configBytes) - // Populate a manifest with the configuration blob and the SquashFS part as the single layer. - layerDescriptor := imgspecv1.Descriptor{ - Digest: imgSrc.blobID, - Size: imgSrc.blobSize, - MediaType: imgspecv1.MediaTypeImageLayerGzip, - } manifest := imgspecv1.Manifest{ - Versioned: imgspecs.Versioned{ - SchemaVersion: 2, - }, + Versioned: imgspecs.Versioned{SchemaVersion: 2}, + MediaType: imgspecv1.MediaTypeImageManifest, Config: imgspecv1.Descriptor{ - Digest: configID, - Size: configSize, + Digest: configDigest, + Size: int64(len(configBytes)), MediaType: imgspecv1.MediaTypeImageConfig, }, - Layers: []imgspecv1.Descriptor{layerDescriptor}, + Layers: []imgspecv1.Descriptor{{ + Digest: layerDigest, + Size: layerSize, + MediaType: imgspecv1.MediaTypeImageLayer, + }}, } manifestBytes, err := json.Marshal(&manifest) if err != nil { - return nil, fmt.Errorf("error generating manifest for %q: %v", ref.resolvedFile, err) + return nil, fmt.Errorf("generating manifest for %q: %w", ref.resolvedFile, err) } + succeeded = true return &sifImageSource{ - ref: ref, - sifimg: sifimg, - workdir: workdir, - diffID: imgSrc.diffID, - diffSize: imgSrc.diffSize, - blobID: imgSrc.blobID, - blobSize: imgSrc.blobSize, - blobType: layerDescriptor.MediaType, - blobFile: imgSrc.blobFile, - config: configBytes, - configID: configID, - configSize: configSize, - manifest: manifestBytes, + ref: ref, + workDir: workDir, + layerDigest: layerDigest, + layerSize: layerSize, + layerFile: layerPath, + config: configBytes, + configDigest: configDigest, + manifest: manifestBytes, }, nil } @@ -222,31 +156,30 @@ func (s *sifImageSource) Reference() types.ImageReference { // Close removes resources associated with an initialized ImageSource, if any. func (s *sifImageSource) Close() error { - os.RemoveAll(s.workdir) - return s.sifimg.UnloadSIFImage() + return os.RemoveAll(s.workDir) } // HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. func (s *sifImageSource) HasThreadSafeGetBlob() bool { - return false + return true } // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. func (s *sifImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) { - // We should only be asked about things in the manifest. Maybe the configuration blob. - if info.Digest == s.configID { - return ioutil.NopCloser(bytes.NewBuffer(s.config)), s.configSize, nil - } - if info.Digest == s.blobID { - reader, err := os.Open(s.blobFile) + switch info.Digest { + case s.configDigest: + return ioutil.NopCloser(bytes.NewBuffer(s.config)), int64(len(s.config)), nil + case s.layerDigest: + reader, err := os.Open(s.layerFile) if err != nil { - return nil, -1, fmt.Errorf("error opening %q: %v", s.blobFile, err) + return nil, -1, fmt.Errorf("opening %q: %w", s.layerFile, err) } - return reader, s.blobSize, nil + return reader, s.layerSize, nil + default: + return nil, -1, fmt.Errorf("no blob with digest %q found", info.Digest.String()) } - return nil, -1, fmt.Errorf("no blob with digest %q found", info.Digest.String()) } // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). @@ -255,7 +188,7 @@ func (s *sifImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache // this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). func (s *sifImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { if instanceDigest != nil { - return nil, "", fmt.Errorf("manifest lists are not supported by the sif transport") + return nil, "", errors.New("manifest lists are not supported by the sif transport") } return s.manifest, imgspecv1.MediaTypeImageManifest, nil } @@ -266,7 +199,7 @@ func (s *sifImageSource) GetManifest(ctx context.Context, instanceDigest *digest // (e.g. if the source never returns manifest lists). func (s *sifImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { if instanceDigest != nil { - return nil, fmt.Errorf("manifest lists are not supported by the sif transport") + return nil, errors.New("manifest lists are not supported by the sif transport") } return nil, nil } diff --git a/sif/transport.go b/sif/transport.go index 3b53f85c0f..18d894bc35 100644 --- a/sif/transport.go +++ b/sif/transport.go @@ -1,8 +1,10 @@ -package sifimage +package sif import ( "context" + "errors" "fmt" + "path/filepath" "strings" "github.com/containers/image/v5/directory/explicitfilepath" @@ -10,7 +12,6 @@ import ( "github.com/containers/image/v5/image" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" - "github.com/pkg/errors" ) func init() { @@ -22,12 +23,6 @@ var Transport = sifTransport{} type sifTransport struct{} -// sifReference is an ImageReference for SIF images. -type sifReference struct { - file string // As specified by the user. May be relative, contain symlinks, etc. - resolvedFile string // Absolute file path with no symlinks, at least at the time of its creation. Primarily used for policy namespaces. -} - func (t sifTransport) Name() string { return "sif" } @@ -37,8 +32,47 @@ func (t sifTransport) ParseReference(reference string) (types.ImageReference, er return NewReference(reference) } +// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys +// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value). +// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion. +// scope passed to this function will not be "", that value is always allowed. +func (t sifTransport) ValidatePolicyConfigurationScope(scope string) error { + if !strings.HasPrefix(scope, "/") { + return fmt.Errorf("Invalid scope %s: Must be an absolute path", scope) + } + // Refuse also "/", otherwise "/" and "" would have the same semantics, + // and "" could be unexpectedly shadowed by the "/" entry. + if scope == "/" { + return errors.New(`Invalid scope "/": Use the generic default scope ""`) + } + cleaned := filepath.Clean(scope) + if cleaned != scope { + return fmt.Errorf(`Invalid scope %s: Uses non-canonical format, perhaps try %s`, scope, cleaned) + } + return nil +} + +// sifReference is an ImageReference for SIF images. +type sifReference struct { + // Note that the interpretation of paths below depends on the underlying filesystem state, which may change under us at any time! + // Either of the paths may point to a different, or no, inode over time. resolvedFile may contain symbolic links, and so on. + + // Generally we follow the intent of the user, and use the "file" member for filesystem operations (e.g. the user can use a relative path to avoid + // being exposed to symlinks and renames in the parent directories to the working directory). + // (But in general, we make no attempt to be completely safe against concurrent hostile filesystem modifications.) + file string // As specified by the user. May be relative, contain symlinks, etc. + resolvedFile string // Absolute file path with no symlinks, at least at the time of its creation. Primarily used for policy namespaces. +} + +// There is no sif.ParseReference because it is rather pointless. +// Callers who need a transport-independent interface will go through +// sifTransport.ParseReference; callers who intentionally deal with SIF files +// can use sif.NewReference. + // NewReference returns an image file reference for a specified path. func NewReference(file string) (types.ImageReference, error) { + // We do not expose an API supplying the resolvedFile; we could, but recomputing it + // is generally cheap enough that we prefer being confident about the properties of resolvedFile. resolved, err := explicitfilepath.ResolvePathToFullyExplicit(file) if err != nil { return nil, err @@ -46,47 +80,56 @@ func NewReference(file string) (types.ImageReference, error) { return sifReference{file: file, resolvedFile: resolved}, nil } -// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys -// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value). -// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion. -// scope passed to this function will not be "", that value is always allowed. -func (t sifTransport) ValidatePolicyConfigurationScope(scope string) error { - return errors.New(`sif: does not support any scopes except the default "" one`) -} - func (ref sifReference) Transport() types.ImageTransport { return Transport } // StringWithinTransport returns a string representation of the reference, which MUST be such that // reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference. +// NOTE: The returned string is not promised to be equal to the original input to ParseReference; +// e.g. default attribute values omitted by the user may be filled in in the return value, or vice versa. +// WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix; +// instead, see transports.ImageName(). func (ref sifReference) StringWithinTransport() string { return ref.file } // DockerReference returns a Docker reference associated with this reference +// (fully explicit, i.e. !reference.IsNameOnly, but reflecting user intent, +// not e.g. after redirect or alias processing), or nil if unknown/not applicable. func (ref sifReference) DockerReference() reference.Named { return nil } // PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup. +// This MUST reflect user intent, not e.g. after processing of third-party redirects or aliases; +// The value SHOULD be fully explicit about its semantics, with no hidden defaults, AND canonical +// (i.e. various references with exactly the same semantics should return the same configuration identity) +// It is fine for the return value to be equal to StringWithinTransport(), and it is desirable but +// not required/guaranteed that it will be a valid input to Transport().ParseReference(). +// Returns "" if configuration identities for these references are not supported. func (ref sifReference) PolicyConfigurationIdentity() string { return ref.resolvedFile } // PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search -// for if explicit configuration for PolicyConfigurationIdentity() is not set +// for if explicit configuration for PolicyConfigurationIdentity() is not set. The list will be processed +// in order, terminating on first match, and an implicit "" is always checked at the end. +// It is STRONGLY recommended for the first element, if any, to be a prefix of PolicyConfigurationIdentity(), +// and each following element to be a prefix of the element preceding it. func (ref sifReference) PolicyConfigurationNamespaces() []string { res := []string{} path := ref.resolvedFile for { lastSlash := strings.LastIndex(path, "/") - if lastSlash == -1 || path == "/" { + if lastSlash == -1 || lastSlash == 0 { break } - res = append(res, path) path = path[:lastSlash] + res = append(res, path) } + // Note that we do not include "/"; it is redundant with the default "" global default, + // and rejected by sifTransport.ValidatePolicyConfigurationScope above. return res } @@ -109,11 +152,13 @@ func (ref sifReference) NewImageSource(ctx context.Context, sys *types.SystemCon return newImageSource(ctx, sys, ref) } +// NewImageDestination returns a types.ImageDestination for this reference. +// The caller must call .Close() on the returned ImageDestination. func (ref sifReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) { - return nil, fmt.Errorf(`"sif:" locations can only be read from, not written to`) + return nil, errors.New(`"sif:" locations can only be read from, not written to`) } // DeleteImage deletes the named image from the registry, if supported. func (ref sifReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error { - return errors.Errorf("Deleting images not implemented for sif: images") + return errors.New("Deleting images not implemented for sif: images") } diff --git a/sif/transport_test.go b/sif/transport_test.go new file mode 100644 index 0000000000..3ac6b79bde --- /dev/null +++ b/sif/transport_test.go @@ -0,0 +1,177 @@ +package sif + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + _ "github.com/containers/image/v5/internal/testing/explicitfilepath-tmpdir" + "github.com/containers/image/v5/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTransportName(t *testing.T) { + assert.Equal(t, "sif", Transport.Name()) +} + +func TestTransportParseReference(t *testing.T) { + testNewReference(t, Transport.ParseReference) +} + +func TestTransportValidatePolicyConfigurationScope(t *testing.T) { + for _, scope := range []string{ + "/etc/passwd", + "/this/does/not/exist", + } { + err := Transport.ValidatePolicyConfigurationScope(scope) + assert.NoError(t, err, scope) + } + + for _, scope := range []string{ + "relative/path", + "/double//slashes", + "/has/./dot", + "/has/dot/../dot", + "/trailing/slash/", + "/", + } { + err := Transport.ValidatePolicyConfigurationScope(scope) + assert.Error(t, err, scope) + } +} + +func TestNewReference(t *testing.T) { + testNewReference(t, NewReference) +} + +// testNewReference is a test shared for Transport.ParseReference and NewReference. +func testNewReference(t *testing.T, fn func(string) (types.ImageReference, error)) { + tmpDir, err := ioutil.TempDir("", "sif-transport-test") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + tmpFile := filepath.Join(tmpDir, "image.sif") + err = ioutil.WriteFile(tmpFile, nil, 0600) + require.NoError(t, err) + + for _, file := range []string{ + "/dev/null", + tmpFile, + "relativepath", + tmpDir + "/thisdoesnotexist", + } { + ref, err := fn(file) + require.NoError(t, err, file) + sifRef, ok := ref.(sifReference) + require.True(t, ok) + assert.Equal(t, file, sifRef.file, file) + } + + _, err = fn(tmpDir + "/thisparentdoesnotexist/something") + assert.Error(t, err) +} + +// refToTempFile creates a temporary file and returns a reference to it. +// The caller should +// defer os.Remove(tmpFile) +func refToTempFile(t *testing.T) (ref types.ImageReference, tmpDir string) { + f, err := ioutil.TempFile("", "sif-transport-test") + require.NoError(t, err) + tmpFile := f.Name() + err = f.Close() + require.NoError(t, err) + ref, err = NewReference(tmpFile) + require.NoError(t, err) + return ref, tmpFile +} + +func TestReferenceTransport(t *testing.T) { + ref, tmpFile := refToTempFile(t) + defer os.Remove(tmpFile) + assert.Equal(t, Transport, ref.Transport()) +} + +func TestReferenceStringWithinTransport(t *testing.T) { + ref, tmpFile := refToTempFile(t) + defer os.Remove(tmpFile) + assert.Equal(t, tmpFile, ref.StringWithinTransport()) +} + +func TestReferenceDockerReference(t *testing.T) { + ref, tmpFile := refToTempFile(t) + defer os.Remove(tmpFile) + assert.Nil(t, ref.DockerReference()) +} + +func TestReferencePolicyConfigurationIdentity(t *testing.T) { + ref, tmpFile := refToTempFile(t) + defer os.Remove(tmpFile) + + assert.Equal(t, tmpFile, ref.PolicyConfigurationIdentity()) + // A non-canonical path. Test just one, the various other cases are + // tested in explicitfilepath.ResolvePathToFullyExplicit. + ref, err := NewReference("/./" + tmpFile) + require.NoError(t, err) + assert.Equal(t, tmpFile, ref.PolicyConfigurationIdentity()) +} + +func TestReferencePolicyConfigurationNamespaces(t *testing.T) { + ref, tmpFile := refToTempFile(t) + defer os.Remove(tmpFile) + // We don't really know enough to make a full equality test here. + ns := ref.PolicyConfigurationNamespaces() + require.NotNil(t, ns) + assert.NotEmpty(t, ns) + assert.Equal(t, filepath.Dir(tmpFile), ns[0]) + + // Test with a known path where the directory should exist. Test just one non-canonical + // path, the various other cases are tested in explicitfilepath.ResolvePathToFullyExplicit. + for _, path := range []string{"/usr/share/probablydoesnotexist.sif", "/usr/share/././probablydoesnoexist.sif"} { + _, err := os.Lstat(filepath.Dir(path)) + require.NoError(t, err) + ref, err := NewReference(path) + require.NoError(t, err) + ns := ref.PolicyConfigurationNamespaces() + require.NotNil(t, ns) + assert.Equal(t, []string{"/usr/share", "/usr"}, ns) + } + + // "/" as a corner case. + ref, err := NewReference("/") + require.NoError(t, err) + assert.Equal(t, []string{}, ref.PolicyConfigurationNamespaces()) +} + +func TestReferenceNewImage(t *testing.T) { + ref, tmpFile := refToTempFile(t) + defer os.Remove(tmpFile) + // A pretty pointless smoke test for now; + // we don't want to require every developer of c/image to have fakeroot etc. around. + _, err := ref.NewImage(context.Background(), nil) + assert.Error(t, err) // Empty file is not valid +} + +func TestReferenceNewImageSource(t *testing.T) { + ref, tmpFile := refToTempFile(t) + defer os.Remove(tmpFile) + // A pretty pointless smoke test for now; + // we don't want to require every developer of c/image to have fakeroot etc. around. + _, err := ref.NewImageSource(context.Background(), nil) + assert.Error(t, err) // Empty file is not valid +} + +func TestReferenceNewImageDestination(t *testing.T) { + ref, tmpFile := refToTempFile(t) + defer os.Remove(tmpFile) + _, err := ref.NewImageDestination(context.Background(), nil) + assert.Error(t, err) +} + +func TestReferenceDeleteImage(t *testing.T) { + ref, tmpFile := refToTempFile(t) + defer os.Remove(tmpFile) + err := ref.DeleteImage(context.Background(), nil) + assert.Error(t, err) +}