diff --git a/go.mod b/go.mod index 917bad4231..b4a0ab3c0c 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/opencontainers/selinux v1.9.1 github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 github.com/pkg/errors v0.9.1 + github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 github.com/ulikunitz/xz v0.5.10 diff --git a/go.sum b/go.sum index 518ee7a89c..b8f9be8c89 100644 --- a/go.sum +++ b/go.sum @@ -603,6 +603,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= diff --git a/sif/internal/sif_util.go b/sif/internal/sif_util.go new file mode 100644 index 0000000000..7bdbdc3884 --- /dev/null +++ b/sif/internal/sif_util.go @@ -0,0 +1,238 @@ +// +build linux + +package internal + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/containers/image/v5/sif/sif" + "github.com/pkg/errors" + + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +type SifImage struct { + fimg sif.FileImage + rootfs *sif.Descriptor + deffile *sif.Descriptor + defReader *io.SectionReader + cmdlist []string + runscript *bytes.Buffer + env *sif.Descriptor + envReader *io.SectionReader + envlist []string +} + +func LoadSIFImage(path string) (image SifImage, err error) { + // open up the SIF file and get its header + image.fimg, err = sif.LoadContainer(path, true) + if err != nil { + return + } + + // check for a system partition and save it + image.rootfs, _, err = image.fimg.GetPartPrimSys() + if err != nil { + return SifImage{}, errors.Wrap(err, "looking up rootfs from SIF file") + } + + // look for a definition file object + searchDesc := sif.Descriptor{Datatype: sif.DataDeffile} + resultDescs, _, err := image.fimg.GetFromDescr(searchDesc) + if err == nil && resultDescs != nil { + // we assume in practice that typical SIF files don't hold multiple deffiles + image.deffile = resultDescs[0] + image.defReader = io.NewSectionReader(image.fimg.Fp, image.deffile.Fileoff, image.deffile.Filelen) + } + if err = image.generateConfig(); err != nil { + return SifImage{}, err + } + + // look for an environment variable set object + searchDesc = sif.Descriptor{Datatype: sif.DataEnvVar} + resultDescs, _, err = image.fimg.GetFromDescr(searchDesc) + if err == nil && resultDescs != nil { + // we assume in practice that typical SIF files don't hold multiple EnvVar sets + image.env = resultDescs[0] + image.envReader = io.NewSectionReader(image.fimg.Fp, image.env.Fileoff, image.env.Filelen) + } + + return image, nil +} + +func (image *SifImage) parseEnvironment(scanner *bufio.Scanner) error { + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || strings.HasPrefix(s, "#") { + continue + } + if strings.HasPrefix(s, "%") { + return nil + } + image.envlist = append(image.envlist, s) + } + if err := scanner.Err(); err != nil { + return errors.Wrap(err, "parsing environment from SIF definition file object") + } + return nil +} + +func (image *SifImage) parseRunscript(scanner *bufio.Scanner) error { + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(s, "%") { + return nil + } + image.cmdlist = append(image.cmdlist, s) + } + if err := scanner.Err(); err != nil { + return errors.Wrap(err, "parsing runscript from SIF definition file object") + } + return nil +} + +func (image *SifImage) generateRunscript() error { + base := `#!/bin/bash +` + image.runscript = bytes.NewBufferString(base) + for _, s := range image.envlist { + _, err := image.runscript.WriteString(fmt.Sprintln(s)) + if err != nil { + return errors.Wrap(err, "writing to runscript buffer") + } + } + for _, s := range image.cmdlist { + _, err := image.runscript.WriteString(fmt.Sprintln(s)) + if err != nil { + return errors.Wrap(err, "writing to runscript buffer") + } + } + return nil +} + +func (image *SifImage) generateConfig() error { + if image.deffile == nil { + image.cmdlist = append(image.cmdlist, "bash") + return nil + } + + // extract %environment/%runscript from definition file + var err error + scanner := bufio.NewScanner(image.defReader) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + again: + if s == `%environment` { + if err = image.parseEnvironment(scanner); err != nil { + return err + } + } else if s == `%runscript` { + if err = image.parseRunscript(scanner); err != nil { + return err + } + } + s = strings.TrimSpace(scanner.Text()) + if s == `%environment` || s == `%runscript` { + goto again + } + } + if err := scanner.Err(); err != nil { + return errors.Wrap(err, "reading lines from SIF definition file object") + } + + if len(image.cmdlist) == 0 && len(image.envlist) == 0 { + image.cmdlist = append(image.cmdlist, "bash") + } else { + if err = image.generateRunscript(); err != nil { + return errors.Wrap(err, "generating runscript") + } + image.cmdlist = []string{"/podman/runscript"} + } + + return nil +} + +func (image SifImage) GetConfig(config *imgspecv1.Image) error { + config.Config.Cmd = append(config.Config.Cmd, image.cmdlist...) + return nil +} + +func (image SifImage) UnloadSIFImage() (err error) { + err = image.fimg.UnloadContainer() + return +} + +func (image SifImage) GetSIFID() string { + return image.fimg.Header.ID.String() +} + +func (image SifImage) GetSIFArch() string { + return sif.GetGoArch(string(image.fimg.Header.Arch[:sif.HdrArchLen-1])) +} + +const squashFilename = "rootfs.squashfs" +const tarFilename = "rootfs.tar" + +func runUnSquashFSTar(tempdir string) (err error) { + script := ` +#!/bin/sh +unsquashfs -f ` + squashFilename + ` && tar --acls --xattrs -C ./squashfs-root -cpf ` + tarFilename + ` ./ +` + + if err = ioutil.WriteFile(filepath.Join(tempdir, "script"), []byte(script), 0755); err != nil { + return err + } + cmd := []string{"fakeroot", "--", "./script"} + + xcmd := exec.Command(cmd[0], cmd[1:]...) + xcmd.Stderr = os.Stderr + xcmd.Dir = tempdir + err = xcmd.Run() + return +} + +func (image *SifImage) writeRunscript(tempdir string) (err error) { + if image.runscript == nil { + return nil + } + rsPath := filepath.Join(tempdir, "squashfs-root", "podman") + if err = os.MkdirAll(rsPath, 0755); err != nil { + return + } + if err = ioutil.WriteFile(filepath.Join(rsPath, "runscript"), image.runscript.Bytes(), 0755); err != nil { + return errors.Wrap(err, "writing /podman/runscript") + } + return nil +} + +func (image SifImage) SquashFSToTarLayer(tempdir string) (tarpath string, err error) { + if _, err = image.fimg.Fp.Seek(image.rootfs.Fileoff, 0); err != nil { + return + } + f, err := os.Create(filepath.Join(tempdir, squashFilename)) + if err != nil { + return + } + defer f.Close() + if _, err = io.CopyN(f, image.fimg.Fp, image.rootfs.Filelen); err != nil { + return + } + if err = f.Sync(); err != nil { + return + } + if err = image.writeRunscript(tempdir); err != nil { + return + } + if err = runUnSquashFSTar(tempdir); err != nil { + return + } + return filepath.Join(tempdir, tarFilename), nil +} diff --git a/sif/sif/LICENSE.md b/sif/sif/LICENSE.md new file mode 100644 index 0000000000..0686bf12c2 --- /dev/null +++ b/sif/sif/LICENSE.md @@ -0,0 +1,27 @@ +Copyright (c) 2018, Sylabs Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/sif/sif/create.go b/sif/sif/create.go new file mode 100644 index 0000000000..b9888f43ad --- /dev/null +++ b/sif/sif/create.go @@ -0,0 +1,575 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +// Copyright (c) 2017, Yannick Cote All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// +build linux + +package sif + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "fmt" + "io" + "os" + "os/user" + "path" + "strconv" + "time" +) + +// Find next offset aligned to block size +func nextAligned(offset int64, align int) int64 { + align64 := uint64(align) + offset64 := uint64(offset) + + if offset64%align64 != 0 { + offset64 = (offset64 & ^(align64 - 1)) + align64 + } + + return int64(offset64) +} + +// Set file pointer offset to next aligned block +func setFileOffNA(fimg *FileImage, alignment int) (int64, error) { + offset, err := fimg.Fp.Seek(0, 1) // get current position + if err != nil { + return -1, fmt.Errorf("seek() getting current file position: %s", err) + } + aligned := nextAligned(offset, alignment) + offset, err = fimg.Fp.Seek(aligned, 0) // set new position + if err != nil { + return -1, fmt.Errorf("seek() getting current file position: %s", err) + } + return offset, nil +} + +// Get current user and returns both uid and gid +func getUserIDs() (int64, int64, error) { + u, err := user.Current() + if err != nil { + return -1, -1, fmt.Errorf("getting current user info: %s", err) + } + + uid, err := strconv.Atoi(u.Uid) + if err != nil { + return -1, -1, fmt.Errorf("converting UID: %s", err) + } + + gid, err := strconv.Atoi(u.Gid) + if err != nil { + return -1, -1, fmt.Errorf("converting GID: %s", err) + } + + return int64(uid), int64(gid), nil +} + +// Fill all of the fields of a Descriptor +func fillDescriptor(fimg *FileImage, index int, input DescriptorInput) (err error) { + descr := &fimg.DescrArr[index] + + curoff, err := fimg.Fp.Seek(0, 1) + if err != nil { + return fmt.Errorf("while file pointer look at: %s", err) + } + + descr.Datatype = input.Datatype + descr.ID = uint32(index) + 1 + descr.Used = true + descr.Groupid = input.Groupid + descr.Link = input.Link + align := os.Getpagesize() + if input.Alignment != 0 { + align = input.Alignment + } + descr.Fileoff, err = setFileOffNA(fimg, align) + if err != nil { + return + } + descr.Filelen = input.Size + descr.Storelen = descr.Fileoff + descr.Filelen - curoff + descr.Ctime = time.Now().Unix() + descr.Mtime = time.Now().Unix() + descr.UID, descr.Gid, err = getUserIDs() + if err != nil { + return fmt.Errorf("filling descriptor: %s", err) + } + descr.SetName(path.Base(input.Fname)) + descr.SetExtra(input.Extra.Bytes()) + + // Check that none or only 1 primary partition is ever set + if descr.Datatype == DataPartition { + ptype, err := descr.GetPartType() + if err != nil { + return err + } + if ptype == PartPrimSys { + if fimg.PrimPartID != 0 { + return fmt.Errorf("only 1 FS data object may be a primary partition") + } + fimg.PrimPartID = descr.ID + arch, err := descr.GetArch() + if err != nil { + return err + } + copy(fimg.Header.Arch[:], arch[:]) + } + } + + return +} + +// Write new data object to the SIF file +func writeDataObject(fimg *FileImage, index int, input DescriptorInput) error { + // if we have bytes in input.data use that instead of an input file + if input.Data != nil { + if _, err := fimg.Fp.Write(input.Data); err != nil { + return fmt.Errorf("copying data object data to SIF file: %s", err) + } + } else { + if n, err := io.Copy(fimg.Fp, input.Fp); err != nil { + return fmt.Errorf("copying data object file to SIF file: %s", err) + } else if n != input.Size && input.Size != 0 { + return fmt.Errorf("short write while copying to SIF file") + } else if input.Size == 0 { + // coming in from os.Stdin (pipe) + descr := &fimg.DescrArr[index] + descr.Filelen = n + descr.SetName("pipe" + fmt.Sprint(index+1)) + } + } + + return nil +} + +// Find a free descriptor and create a memory representation for addition to the SIF file +func createDescriptor(fimg *FileImage, input DescriptorInput) (err error) { + var ( + idx int + v Descriptor + ) + + if fimg.Header.Dfree == 0 { + return fmt.Errorf("no descriptor table free entry") + } + + // look for a free entry in the descriptor table + for idx, v = range fimg.DescrArr { + if !v.Used { + break + } + } + if int64(idx) == fimg.Header.Dtotal-1 && fimg.DescrArr[idx].Used { + return fmt.Errorf("no descriptor table free entry, warning: header.Dfree was > 0") + } + + // fill in SIF file descriptor + if err = fillDescriptor(fimg, idx, input); err != nil { + return + } + + // write data object associated to the descriptor in SIF file + if err = writeDataObject(fimg, idx, input); err != nil { + return fmt.Errorf("writing data object for SIF file: %s", err) + } + + // update some global header fields from adding this new descriptor + fimg.Header.Dfree-- + fimg.Header.Datalen += fimg.DescrArr[idx].Storelen + + return +} + +// Release and write the data object descriptor to backing storage (SIF container file) +func writeDescriptors(fimg *FileImage) error { + // first, move to descriptor start offset + if _, err := fimg.Fp.Seek(DescrStartOffset, 0); err != nil { + return fmt.Errorf("seeking to descriptor start offset: %s", err) + } + + for _, v := range fimg.DescrArr { + if err := binary.Write(fimg.Fp, binary.LittleEndian, v); err != nil { + return fmt.Errorf("binary writing descrtable to buf: %s", err) + } + } + fimg.Header.Descrlen = int64(binary.Size(fimg.DescrArr)) + + return nil +} + +// Write the global header to file +func writeHeader(fimg *FileImage) error { + // first, move to descriptor start offset + if _, err := fimg.Fp.Seek(0, 0); err != nil { + return fmt.Errorf("seeking to beginning of the file: %s", err) + } + + if err := binary.Write(fimg.Fp, binary.LittleEndian, fimg.Header); err != nil { + return fmt.Errorf("binary writing header to buf: %s", err) + } + + return nil +} + +// CreateContainer is responsible for the creation of a new SIF container +// file. It takes the creation information specification as input +// and produces an output file as specified in the input data. +func CreateContainer(cinfo CreateInfo) (fimg *FileImage, err error) { + fimg = &FileImage{} + fimg.DescrArr = make([]Descriptor, DescrNumEntries) + + // Prepare a fresh global header + copy(fimg.Header.Launch[:], cinfo.Launchstr) + copy(fimg.Header.Magic[:], HdrMagic) + copy(fimg.Header.Version[:], cinfo.Sifversion) + copy(fimg.Header.Arch[:], HdrArchUnknown) + copy(fimg.Header.ID[:], cinfo.ID[:]) + fimg.Header.Ctime = time.Now().Unix() + fimg.Header.Mtime = time.Now().Unix() + fimg.Header.Dfree = DescrNumEntries + fimg.Header.Dtotal = DescrNumEntries + fimg.Header.Descroff = DescrStartOffset + fimg.Header.Dataoff = DataStartOffset + + // Create container file + fimg.Fp, err = os.OpenFile(cinfo.Pathname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return nil, fmt.Errorf("container file creation failed: %s", err) + } + defer fimg.Fp.Close() + + // set file pointer to start of data section */ + if _, err = fimg.Fp.Seek(DataStartOffset, 0); err != nil { + return nil, fmt.Errorf("setting file offset pointer to DataStartOffset: %s", err) + } + + for _, v := range cinfo.InputDescr { + if err = createDescriptor(fimg, v); err != nil { + return + } + } + + // Write down the descriptor array + if err = writeDescriptors(fimg); err != nil { + return + } + + // Write down global header to file + if err = writeHeader(fimg); err != nil { + return + } + + return +} + +func zeroData(fimg *FileImage, descr *Descriptor) error { + // first, move to data object offset + if _, err := fimg.Fp.Seek(descr.Fileoff, 0); err != nil { + return fmt.Errorf("seeking to data object offset: %s", err) + } + + var zero [4096]byte + n := descr.Filelen + upbound := int64(4096) + for { + if n < 4096 { + upbound = n + } + + if _, err := fimg.Fp.Write(zero[:upbound]); err != nil { + return fmt.Errorf("writing 0's to data object") + } + n -= 4096 + if n <= 0 { + break + } + } + + return nil +} + +func resetDescriptor(fimg *FileImage, index int) error { + // If we remove the primary partition, set the global header Arch field to HdrArchUnknown + // to indicate that the SIF file doesn't include a primary partition and no dependency + // on any architecture exists. + _, idx, _ := fimg.GetPartPrimSys() + if idx == index { + fimg.PrimPartID = 0 + copy(fimg.Header.Arch[:], HdrArchUnknown) + } + + offset := fimg.Header.Descroff + int64(index)*int64(binary.Size(fimg.DescrArr[0])) + + // first, move to descriptor offset + if _, err := fimg.Fp.Seek(offset, 0); err != nil { + return fmt.Errorf("seeking to descriptor: %s", err) + } + + var emptyDesc Descriptor + if err := binary.Write(fimg.Fp, binary.LittleEndian, emptyDesc); err != nil { + return fmt.Errorf("binary writing empty descriptor: %s", err) + } + + return nil +} + +// AddObject add a new data object and its descriptor into the specified SIF file. +func (fimg *FileImage) AddObject(input DescriptorInput) error { + // set file pointer to the end of data section + if _, err := fimg.Fp.Seek(fimg.Header.Dataoff+fimg.Header.Datalen, 0); err != nil { + return fmt.Errorf("setting file offset pointer to DataStartOffset: %s", err) + } + + // create a new descriptor entry from input data + if err := createDescriptor(fimg, input); err != nil { + return err + } + + // write down the descriptor array + if err := writeDescriptors(fimg); err != nil { + return err + } + + fimg.Header.Mtime = time.Now().Unix() + // write down global header to file + if err := writeHeader(fimg); err != nil { + return err + } + + if err := fimg.Fp.Sync(); err != nil { + return fmt.Errorf("while sync'ing new data object to SIF file: %s", err) + } + + return nil +} + +// descrIsLast return true if passed descriptor's object is the last in a SIF file +func objectIsLast(fimg *FileImage, descr *Descriptor) bool { + return fimg.Filesize == descr.Fileoff+descr.Filelen +} + +// compactAtDescr joins data objects leading and following "descr" by compacting a SIF file +func compactAtDescr(fimg *FileImage, descr *Descriptor) error { + var prev Descriptor + + for _, v := range fimg.DescrArr { + if !v.Used || v.ID == descr.ID { + continue + } else { + if v.Fileoff > prev.Fileoff { + prev = v + } + } + } + // make sure it's not the only used descriptor first + if prev.Used { + if err := fimg.Fp.Truncate(prev.Fileoff + prev.Filelen); err != nil { + return err + } + } else { + if err := fimg.Fp.Truncate(descr.Fileoff); err != nil { + return err + } + } + fimg.Header.Datalen -= descr.Storelen + return nil +} + +// DeleteObject removes data from a SIF file referred to by id. The descriptor for the +// data object is free'd and can be reused later. There's currenly 2 clean mode specified +// by flags: DelZero, to zero out the data region for security and DelCompact to +// remove and shink the file compacting the unused area. +func (fimg *FileImage) DeleteObject(id uint32, flags int) error { + descr, index, err := fimg.GetFromDescrID(id) + if err != nil { + return err + } + + switch flags { + case DelZero: + if err = zeroData(fimg, descr); err != nil { + return err + } + case DelCompact: + if objectIsLast(fimg, descr) { + if err = compactAtDescr(fimg, descr); err != nil { + return err + } + } else { + return fmt.Errorf("method (DelCompact) not implemented yet") + } + default: + if objectIsLast(fimg, descr) { + if err = compactAtDescr(fimg, descr); err != nil { + return err + } + } + } + + // update some global header fields from deleting this descriptor + fimg.Header.Dfree++ + fimg.Header.Mtime = time.Now().Unix() + + // zero out the unused descriptor + if err = resetDescriptor(fimg, index); err != nil { + return err + } + + // update global header + if err = writeHeader(fimg); err != nil { + return err + } + + if err := fimg.Fp.Sync(); err != nil { + return fmt.Errorf("while sync'ing deleted data object to SIF file: %s", err) + } + + return nil +} + +// SetPartExtra serializes the partition and fs type info into a binary buffer +func (di *DescriptorInput) SetPartExtra(fs Fstype, part Parttype, arch string) error { + extra := Partition{ + Fstype: fs, + Parttype: part, + } + if arch == HdrArchUnknown { + return fmt.Errorf("architecture not supported: %v", arch) + } + copy(extra.Arch[:], arch[:]) + + // serialize the partition data for integration with the base descriptor input + if err := binary.Write(&di.Extra, binary.LittleEndian, extra); err != nil { + return err + } + return nil +} + +// SetSignExtra serializes the hash type and the entity info into a binary buffer +func (di *DescriptorInput) SetSignExtra(hash Hashtype, entity string) error { + extra := Signature{ + Hashtype: hash, + } + + h, err := hex.DecodeString(entity) + if err != nil { + return err + } + copy(extra.Entity[:], h) + + // serialize the signature data for integration with the base descriptor input + if err := binary.Write(&di.Extra, binary.LittleEndian, extra); err != nil { + return err + } + return nil +} + +// SetName sets the byte array field "Name" to the value of string "name" +func (d *Descriptor) SetName(name string) { + copy(d.Name[:], []byte(name)) +} + +// SetExtra sets the extra byte array to a provided byte array +func (d *Descriptor) SetExtra(extra []byte) { + copy(d.Extra[:], extra) +} + +// SetPrimPart sets the specified system partition to be the primary one +func (fimg *FileImage) SetPrimPart(id uint32) error { + descr, _, err := fimg.GetFromDescrID(id) + if err != nil { + return err + } + + if descr.Datatype != DataPartition { + return fmt.Errorf("not a volume partition") + } + + ptype, err := descr.GetPartType() + if err != nil { + return err + } + + // if already primary system partition, nothing to do + if ptype == PartPrimSys { + return nil + } + + if ptype != PartSystem { + return fmt.Errorf("partition must be of system type") + } + + olddescr, _, err := fimg.GetPartPrimSys() + if err != nil && err != ErrNotFound { + return err + } + + fs, err := descr.GetFsType() + if err != nil { + return nil + } + + arch, err := descr.GetArch() + if err != nil { + return err + } + + copy(fimg.Header.Arch[:], arch[:]) + fimg.PrimPartID = descr.ID + + extra := Partition{ + Fstype: fs, + Parttype: PartPrimSys, + } + copy(extra.Arch[:], arch[:]) + + var extrabuf bytes.Buffer + if err := binary.Write(&extrabuf, binary.LittleEndian, extra); err != nil { + return err + } + descr.SetExtra(extrabuf.Bytes()) + + if olddescr != nil { + oldfs, err := olddescr.GetFsType() + if err != nil { + return nil + } + oldarch, err := olddescr.GetArch() + if err != nil { + return nil + } + + oldextra := Partition{ + Fstype: oldfs, + Parttype: PartSystem, + } + copy(oldextra.Arch[:], oldarch[:]) + + var oldextrabuf bytes.Buffer + if err := binary.Write(&oldextrabuf, binary.LittleEndian, oldextra); err != nil { + return err + } + olddescr.SetExtra(oldextrabuf.Bytes()) + } + + // write down the descriptor array + if err := writeDescriptors(fimg); err != nil { + return err + } + + fimg.Header.Mtime = time.Now().Unix() + // write down global header to file + if err := writeHeader(fimg); err != nil { + return err + } + + if err := fimg.Fp.Sync(); err != nil { + return fmt.Errorf("while sync'ing new data object to SIF file: %s", err) + } + + return nil +} diff --git a/sif/sif/init.go b/sif/sif/init.go new file mode 100644 index 0000000000..c5c84504c7 --- /dev/null +++ b/sif/sif/init.go @@ -0,0 +1,23 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +// Copyright (c) 2017, Yannick Cote All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// +build linux + +package sif + +import ( + "bytes" + "log" +) + +var ( + sifLoggerBuf bytes.Buffer + siflog = log.New(&sifLoggerBuf, "", log.Ldate|log.Ltime|log.Lshortfile) +) + +func init() { +} diff --git a/sif/sif/load.go b/sif/sif/load.go new file mode 100644 index 0000000000..610f5af66f --- /dev/null +++ b/sif/sif/load.go @@ -0,0 +1,227 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +// Copyright (c) 2017, Yannick Cote All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// +build linux + +package sif + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + "syscall" +) + +// Read the global header from the container file +func readHeader(fimg *FileImage) error { + if err := binary.Read(fimg.Reader, binary.LittleEndian, &fimg.Header); err != nil { + return fmt.Errorf("reading global header from container file: %s", err) + } + + return nil +} + +// Read the used descriptors and populate an in-memory representation of those in node list +func readDescriptors(fimg *FileImage) error { + // start by positioning us to the start of descriptors + _, err := fimg.Reader.Seek(fimg.Header.Descroff, 0) + if err != nil { + return fmt.Errorf("seek() setting to descriptors start: %s", err) + } + + // Initialize descriptor array (slice) and read them all from file + fimg.DescrArr = make([]Descriptor, fimg.Header.Dtotal) + if err := binary.Read(fimg.Reader, binary.LittleEndian, &fimg.DescrArr); err != nil { + fimg.DescrArr = nil + return fmt.Errorf("reading descriptor array from container file: %s", err) + } + + descr, _, err := fimg.GetPartPrimSys() + if err == nil { + fimg.PrimPartID = descr.ID + } + + return nil +} + +// Look at key fields from the global header to assess SIF validity. +// `runnable' checks is current container can run on host. +func isValidSif(fimg *FileImage) error { + // check various header fields + if string(fimg.Header.Magic[:HdrMagicLen-1]) != HdrMagic { + return fmt.Errorf("invalid SIF file: Magic |%s| want |%s|", fimg.Header.Magic, HdrMagic) + } + if string(fimg.Header.Version[:HdrVersionLen-1]) > HdrVersion { + return fmt.Errorf("invalid SIF file: Version %s want <= %s", fimg.Header.Version, HdrVersion) + } + + return nil +} + +// mapFile takes a file pointer and returns a slice of bytes representing the file data +func (fimg *FileImage) mapFile(rdonly bool) error { + prot := syscall.PROT_READ + flags := syscall.MAP_PRIVATE + + info, err := fimg.Fp.Stat() + if err != nil { + return fmt.Errorf("while trying to size SIF file to mmap") + } + fimg.Filesize = info.Size() + + size := nextAligned(info.Size(), syscall.Getpagesize()) + if int64(int(size)) < info.Size() { + return fmt.Errorf("file is to big to be mapped") + } + + if !rdonly { + prot = syscall.PROT_WRITE + flags = syscall.MAP_SHARED + } + + fimg.Filedata, err = syscall.Mmap(int(fimg.Fp.Fd()), 0, int(size), prot, flags) + if err != nil { + // mmap failed, use sequential read() instead for top of file + siflog.Printf("mmap on %s failed, reading buffer sequentially...", fimg.Fp.Name()) + fimg.Filedata = make([]byte, DataStartOffset) + + // start by positioning us to the start of the file + _, err := fimg.Fp.Seek(0, 0) + if err != nil { + return fmt.Errorf("seek() setting to start of file: %s", err) + } + + if n, err := fimg.Fp.Read(fimg.Filedata); n != DataStartOffset { + return fmt.Errorf("short read while reading top of file: %v", err) + + } + fimg.Amodebuf = true + } + + // create and associate a new bytes.Reader on top of mmap'ed or buffered data from file + fimg.Reader = bytes.NewReader(fimg.Filedata) + + return nil +} + +func (fimg *FileImage) unmapFile() error { + if fimg.Amodebuf { + return nil + } + if err := syscall.Munmap(fimg.Filedata); err != nil { + return fmt.Errorf("while calling unmapping SIF file") + } + return nil +} + +// LoadContainer is responsible for loading a SIF container file. It takes +// the container file name, and whether the file is opened as read-only +// as arguments. +func LoadContainer(filename string, rdonly bool) (fimg FileImage, err error) { + if rdonly { // open SIF rdonly if mounting immutable partitions or inspecting the image + if fimg.Fp, err = os.Open(filename); err != nil { + return fimg, fmt.Errorf("opening(RDONLY) container file: %s", err) + } + } else { // open SIF read-write when adding and removing data objects + if fimg.Fp, err = os.OpenFile(filename, os.O_RDWR, 0644); err != nil { + return fimg, fmt.Errorf("opening(RDWR) container file: %s", err) + } + } + + // get a memory map of the SIF file + if err = fimg.mapFile(rdonly); err != nil { + return + } + + // read global header from SIF file + if err = readHeader(&fimg); err != nil { + return + } + + // validate global header + if err = isValidSif(&fimg); err != nil { + return + } + + // read descriptor array from SIF file + if err = readDescriptors(&fimg); err != nil { + return + } + + return +} + +// LoadContainerFp is responsible for loading a SIF container file. It takes +// a *os.File pointing to an opened file, and whether the file is opened as +// read-only for arguments. +func LoadContainerFp(fp *os.File, rdonly bool) (fimg FileImage, err error) { + if fp == nil { + return fimg, fmt.Errorf("provided fp for file is invalid") + } + + fimg.Fp = fp + + // get a memory map of the SIF file + if err = fimg.mapFile(rdonly); err != nil { + return + } + + // read global header from SIF file + if err = readHeader(&fimg); err != nil { + return + } + + // validate global header + if err = isValidSif(&fimg); err != nil { + return + } + + // read descriptor array from SIF file + if err = readDescriptors(&fimg); err != nil { + return + } + + return fimg, nil +} + +// LoadContainerReader is responsible for processing SIF data from a byte stream +// and extract various components like the global header, descriptors and even +// perhaps data, depending on how much is read from the source. +func LoadContainerReader(b *bytes.Reader) (fimg FileImage, err error) { + fimg.Reader = b + + // read global header from SIF file + if err = readHeader(&fimg); err != nil { + return + } + + // validate global header + if err = isValidSif(&fimg); err != nil { + return + } + + // in the case where the reader buffer doesn't include descriptor data, we + // don't return an error and DescrArr will be set to nil + _ = readDescriptors(&fimg) + + return fimg, nil +} + +// UnloadContainer closes the SIF container file and free associated resources if needed +func (fimg *FileImage) UnloadContainer() (err error) { + // if SIF data comes from file, not a slice buffer (see LoadContainer() variants) + if fimg.Fp != nil { + if err = fimg.unmapFile(); err != nil { + return + } + if err = fimg.Fp.Close(); err != nil { + return fmt.Errorf("closing SIF file failed, corrupted: don't use: %s", err) + } + } + return +} diff --git a/sif/sif/lookup.go b/sif/sif/lookup.go new file mode 100644 index 0000000000..363050827b --- /dev/null +++ b/sif/sif/lookup.go @@ -0,0 +1,384 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +// Copyright (c) 2017, Yannick Cote All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// +build linux + +package sif + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "strings" +) + +// ErrNotFound is the code for when no search key is not found +var ErrNotFound = errors.New("no match found") + +// ErrMultValues is the code for when search key is not unique +var ErrMultValues = errors.New("lookup would return more than one match") + +// +// Methods on (fimg *FIleImage) +// + +// GetSIFArch returns the SIF arch code from go runtime arch code +func GetSIFArch(goarch string) (sifarch string) { + var ok bool + + archMap := map[string]string{ + "386": HdrArch386, + "amd64": HdrArchAMD64, + "arm": HdrArchARM, + "arm64": HdrArchARM64, + "ppc64": HdrArchPPC64, + "ppc64le": HdrArchPPC64le, + "mips": HdrArchMIPS, + "mipsle": HdrArchMIPSle, + "mips64": HdrArchMIPS64, + "mips64le": HdrArchMIPS64le, + "s390x": HdrArchS390x, + } + + if sifarch, ok = archMap[goarch]; !ok { + sifarch = HdrArchUnknown + } + return sifarch +} + +// GetGoArch returns the go runtime arch code from the SIF arch code +func GetGoArch(sifarch string) (goarch string) { + var ok bool + + archMap := map[string]string{ + HdrArch386: "386", + HdrArchAMD64: "amd64", + HdrArchARM: "arm", + HdrArchARM64: "arm64", + HdrArchPPC64: "ppc64", + HdrArchPPC64le: "ppc64le", + HdrArchMIPS: "mips", + HdrArchMIPSle: "mipsle", + HdrArchMIPS64: "mips64", + HdrArchMIPS64le: "mips64le", + HdrArchS390x: "s390x", + } + + if goarch, ok = archMap[sifarch]; !ok { + goarch = "unknown" + } + return goarch +} + +// GetHeader returns the loaded SIF global header +func (fimg *FileImage) GetHeader() *Header { + return &fimg.Header +} + +// GetFromDescrID searches for a descriptor with +func (fimg *FileImage) GetFromDescrID(id uint32) (*Descriptor, int, error) { + var match = -1 + + for i, v := range fimg.DescrArr { + if !v.Used { + continue + } else { + if v.ID == id { + if match != -1 { + return nil, -1, ErrMultValues + } + match = i + } + } + } + + if match == -1 { + return nil, -1, ErrNotFound + } + + return &fimg.DescrArr[match], match, nil +} + +// GetPartFromGroup searches for partition descriptors inside a specific group +func (fimg *FileImage) GetPartFromGroup(groupid uint32) ([]*Descriptor, []int, error) { + var descrs []*Descriptor + var indexes []int + var count int + + for i, v := range fimg.DescrArr { + if !v.Used { + continue + } else { + if v.Datatype == DataPartition && v.Groupid == groupid { + indexes = append(indexes, i) + descrs = append(descrs, &fimg.DescrArr[i]) + count++ + } + } + } + + if count == 0 { + return nil, nil, ErrNotFound + } + + return descrs, indexes, nil +} + +// GetSignFromGroup searches for signature descriptors inside a specific group +func (fimg *FileImage) GetSignFromGroup(groupid uint32) ([]*Descriptor, []int, error) { + var descrs []*Descriptor + var indexes []int + var count int + + for i, v := range fimg.DescrArr { + if !v.Used { + continue + } else { + if v.Datatype == DataSignature && v.Groupid == groupid { + indexes = append(indexes, i) + descrs = append(descrs, &fimg.DescrArr[i]) + count++ + } + } + } + + if count == 0 { + return nil, nil, ErrNotFound + } + + return descrs, indexes, nil +} + +// GetFromLinkedDescr searches for descriptors that point to "id" +func (fimg *FileImage) GetFromLinkedDescr(ID uint32) ([]*Descriptor, []int, error) { + var descrs []*Descriptor + var indexes []int + var count int + + for i, v := range fimg.DescrArr { + if !v.Used { + continue + } else { + if v.Link == ID { + indexes = append(indexes, i) + descrs = append(descrs, &fimg.DescrArr[i]) + count++ + } + } + } + + if count == 0 { + return nil, nil, ErrNotFound + } + + return descrs, indexes, nil +} + +// GetFromDescr searches for descriptors comparing all non-nil fields of a provided descriptor +func (fimg *FileImage) GetFromDescr(descr Descriptor) ([]*Descriptor, []int, error) { + var descrs []*Descriptor + var indexes []int + var count int + + for i, v := range fimg.DescrArr { + if !v.Used { + continue + } else { + if descr.Datatype != 0 && descr.Datatype != v.Datatype { + continue + } + if descr.ID != 0 && descr.ID != v.ID { + continue + } + if descr.Groupid != 0 && descr.Groupid != v.Groupid { + continue + } + if descr.Link != 0 && descr.Link != v.Link { + continue + } + if descr.Fileoff != 0 && descr.Fileoff != v.Fileoff { + continue + } + if descr.Filelen != 0 && descr.Filelen != v.Filelen { + continue + } + if descr.Storelen != 0 && descr.Storelen != v.Storelen { + continue + } + if descr.Ctime != 0 && descr.Ctime != v.Ctime { + continue + } + if descr.Mtime != 0 && descr.Mtime != v.Mtime { + continue + } + if descr.UID != 0 && descr.UID != v.UID { + continue + } + if descr.Gid != 0 && descr.Gid != v.Gid { + continue + } + if descr.Name[0] != 0 && !bytes.Equal(descr.Name[:], v.Name[:]) { + continue + } + + indexes = append(indexes, i) + descrs = append(descrs, &fimg.DescrArr[i]) + count++ + } + } + + if count == 0 { + return nil, nil, ErrNotFound + } + + return descrs, indexes, nil +} + +// +// Methods on (descr *Descriptor) +// + +// GetData return a memory mapped byte slice mirroring the data object in a SIF file. +func (descr *Descriptor) GetData(fimg *FileImage) []byte { + if fimg.Amodebuf { + _, err := fimg.Fp.Seek(descr.Fileoff, 0) + if err != nil { + return nil + } + data := make([]byte, descr.Filelen) + if n, _ := fimg.Fp.Read(data); int64(n) != descr.Filelen { + return nil + } + return data + } + + return fimg.Filedata[descr.Fileoff : descr.Fileoff+descr.Filelen] +} + +// GetName returns the name tag associated with the descriptor. Analogous to file name. +func (descr *Descriptor) GetName() string { + return strings.TrimRight(string(descr.Name[:]), "\000") +} + +// GetFsType extracts the Fstype field from the Extra field of a Partition Descriptor +func (descr *Descriptor) GetFsType() (Fstype, error) { + if descr.Datatype != DataPartition { + return -1, fmt.Errorf("expected DataPartition, got %v", descr.Datatype) + } + + var pinfo Partition + b := bytes.NewReader(descr.Extra[:]) + if err := binary.Read(b, binary.LittleEndian, &pinfo); err != nil { + return -1, fmt.Errorf("while extracting Partition extra info: %s", err) + } + + return pinfo.Fstype, nil +} + +// GetPartType extracts the Parttype field from the Extra field of a Partition Descriptor +func (descr *Descriptor) GetPartType() (Parttype, error) { + if descr.Datatype != DataPartition { + return -1, fmt.Errorf("expected DataPartition, got %v", descr.Datatype) + } + + var pinfo Partition + b := bytes.NewReader(descr.Extra[:]) + if err := binary.Read(b, binary.LittleEndian, &pinfo); err != nil { + return -1, fmt.Errorf("while extracting Partition extra info: %s", err) + } + + return pinfo.Parttype, nil +} + +// GetArch extracts the Arch field from the Extra field of a Partition Descriptor +func (descr *Descriptor) GetArch() ([HdrArchLen]byte, error) { + if descr.Datatype != DataPartition { + return [HdrArchLen]byte{}, fmt.Errorf("expected DataPartition, got %v", descr.Datatype) + } + + var pinfo Partition + b := bytes.NewReader(descr.Extra[:]) + if err := binary.Read(b, binary.LittleEndian, &pinfo); err != nil { + return [HdrArchLen]byte{}, fmt.Errorf("while extracting Partition extra info: %s", err) + } + + return pinfo.Arch, nil +} + +// GetHashType extracts the Hashtype field from the Extra field of a Signature Descriptor +func (descr *Descriptor) GetHashType() (Hashtype, error) { + if descr.Datatype != DataSignature { + return -1, fmt.Errorf("expected DataSignature, got %v", descr.Datatype) + } + + var sinfo Signature + b := bytes.NewReader(descr.Extra[:]) + if err := binary.Read(b, binary.LittleEndian, &sinfo); err != nil { + return -1, fmt.Errorf("while extracting Signature extra info: %s", err) + } + + return sinfo.Hashtype, nil +} + +// GetEntity extracts the signing entity field from the Extra field of a Signature Descriptor +func (descr *Descriptor) GetEntity() ([]byte, error) { + if descr.Datatype != DataSignature { + return nil, fmt.Errorf("expected DataSignature, got %v", descr.Datatype) + } + + var sinfo Signature + b := bytes.NewReader(descr.Extra[:]) + if err := binary.Read(b, binary.LittleEndian, &sinfo); err != nil { + return nil, fmt.Errorf("while extracting Signature extra info: %s", err) + } + + return sinfo.Entity[:], nil +} + +// GetEntityString returns the string version of the stored entity +func (descr *Descriptor) GetEntityString() (string, error) { + fingerprint, err := descr.GetEntity() + if err != nil { + return "", err + } + + return fmt.Sprintf("%0X", fingerprint[:20]), nil +} + +// GetPartPrimSys returns the primary system partition if present. There should +// be only one primary system partition in a SIF file. +func (fimg *FileImage) GetPartPrimSys() (*Descriptor, int, error) { + var descr *Descriptor + index := -1 + + for i, v := range fimg.DescrArr { + if !v.Used { + continue + } else { + if v.Datatype == DataPartition { + ptype, err := v.GetPartType() + if err != nil { + return nil, -1, err + } + if ptype == PartPrimSys { + if index != -1 { + return nil, -1, ErrMultValues + } + index = i + descr = &fimg.DescrArr[i] + } + } + } + } + + if index == -1 { + return nil, -1, ErrNotFound + } + + return descr, index, nil +} diff --git a/sif/sif/sif.go b/sif/sif/sif.go new file mode 100644 index 0000000000..964d7b72f7 --- /dev/null +++ b/sif/sif/sif.go @@ -0,0 +1,297 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +// Copyright (c) 2017, Yannick Cote All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// +build linux + +// Package sif implements data structures and routines to create +// and access SIF files. +// - sif.go contains the data definition the file format. +// - create.go implements the core functionality for the creation of +// of new SIF files. +// - load.go implements the core functionality for the loading of +// existing SIF files. +// - lookup.go mostly implements search/lookup and printing routines +// and access to specific descriptor/data found in SIF container files. +package sif + +import ( + "bytes" + "os" + + uuid "github.com/satori/go.uuid" +) + +// Layout of a SIF file (example) +// +// .================================================. +// | GLOBAL HEADER: Sifheader | +// | - launch: "#!/usr/bin/env..." | +// | - magic: "SIF_MAGIC" | +// | - version: "1" | +// | - arch: "4" | +// | - uuid: b2659d4e-bd50-4ea5-bd17-eec5e54f918e | +// | - ctime: 1504657553 | +// | - mtime: 1504657653 | +// | - ndescr: 3 | +// | - descroff: 120 | --. +// | - descrlen: 432 | | +// | - dataoff: 4096 | | +// | - datalen: 619362 | | +// |------------------------------------------------| <-' +// | DESCR[0]: Sifdeffile | +// | - Sifcommon | +// | - datatype: DATA_DEFFILE | +// | - id: 1 | +// | - groupid: 1 | +// | - link: NONE | +// | - fileoff: 4096 | --. +// | - filelen: 222 | | +// |------------------------------------------------| <-----. +// | DESCR[1]: Sifpartition | | | +// | - Sifcommon | | | +// | - datatype: DATA_PARTITION | | | +// | - id: 2 | | | +// | - groupid: 1 | | | +// | - link: NONE | | | +// | - fileoff: 4318 | ----. | +// | - filelen: 618496 | | | | +// | - fstype: Squashfs | | | | +// | - parttype: System | | | | +// | - content: Linux | | | | +// |------------------------------------------------| | | | +// | DESCR[2]: Sifsignature | | | | +// | - Sifcommon | | | | +// | - datatype: DATA_SIGNATURE | | | | +// | - id: 3 | | | | +// | - groupid: NONE | | | | +// | - link: 2 | ------' +// | - fileoff: 622814 | ------. +// | - filelen: 644 | | | | +// | - hashtype: SHA384 | | | | +// | - entity: @ | | | | +// |------------------------------------------------| <-' | | +// | Definition file data | | | +// | . | | | +// | . | | | +// | . | | | +// |------------------------------------------------| <---' | +// | File system partition image | | +// | . | | +// | . | | +// | . | | +// |------------------------------------------------| <-----' +// | Signed verification data | +// | . | +// | . | +// | . | +// `================================================' + +// SIF header constants and quantities +const ( + HdrLaunch = "#!/usr/bin/env run-singularity\n" + HdrMagic = "SIF_MAGIC" // SIF identification + HdrVersion = "02" // SIF SPEC VERSION + HdrArchUnknown = "00" // Undefined/Unsupported arch + HdrArch386 = "01" // 386 (i[3-6]86) arch code + HdrArchAMD64 = "02" // AMD64 arch code + HdrArchARM = "03" // ARM arch code + HdrArchARM64 = "04" // AARCH64 arch code + HdrArchPPC64 = "05" // PowerPC 64 arch code + HdrArchPPC64le = "06" // PowerPC 64 little-endian arch code + HdrArchMIPS = "07" // MIPS arch code + HdrArchMIPSle = "08" // MIPS little-endian arch code + HdrArchMIPS64 = "09" // MIPS64 arch code + HdrArchMIPS64le = "10" // MIPS64 little-endian arch code + HdrArchS390x = "11" // IBM s390x arch code + + HdrLaunchLen = 32 // len("#!/usr/bin/env... ") + HdrMagicLen = 10 // len("SIF_MAGIC") + HdrVersionLen = 3 // len("99") + HdrArchLen = 3 // len("99") + + DescrNumEntries = 48 // the default total number of available descriptors + DescrGroupMask = 0xf0000000 // groups start at that offset + DescrUnusedGroup = DescrGroupMask // descriptor without a group + DescrDefaultGroup = DescrGroupMask | 1 // first groupid number created + DescrUnusedLink = 0 // descriptor without link to other + DescrEntityLen = 256 // len("Joe Bloe ...") + DescrNameLen = 128 // descriptor name (string identifier) + DescrMaxPrivLen = 384 // size reserved for descriptor specific data + DescrStartOffset = 4096 // where descriptors start after global header + DataStartOffset = 32768 // where data object start after descriptors +) + +// Datatype represents the different SIF data object types stored in the image +type Datatype int32 + +// List of supported SIF data types +const ( + DataDeffile Datatype = iota + 0x4001 // definition file data object + DataEnvVar // environment variables data object + DataLabels // JSON labels data object + DataPartition // file system data object + DataSignature // signing/verification data object + DataGenericJSON // generic JSON meta-data + DataGeneric // generic / raw data +) + +// Fstype represents the different SIF file system types found in partition data objects +type Fstype int32 + +// List of supported file systems +const ( + FsSquash Fstype = iota + 1 // Squashfs file system, RDONLY + FsExt3 // EXT3 file system, RDWR (deprecated) + FsImmuObj // immutable data object archive + FsRaw // raw data +) + +// Parttype represents the different SIF container partition types (system and data) +type Parttype int32 + +// List of supported partition types +const ( + PartSystem Parttype = iota + 1 // partition hosts an operating system + PartPrimSys // partition hosts the primary operating system + PartData // partition hosts data only + PartOverlay // partition hosts an overlay +) + +// Hashtype represents the different SIF hashing function types used to fingerprint data objects +type Hashtype int32 + +// List of supported hash functions +const ( + HashSHA256 Hashtype = iota + 1 + HashSHA384 + HashSHA512 + HashBLAKE2S + HashBLAKE2B +) + +// SIF data object deletation strategies +const ( + DelZero = iota + 1 // zero the data object bytes + DelCompact // free the space used by data object +) + +// Descriptor represents the SIF descriptor type +type Descriptor struct { + Datatype Datatype // informs of descriptor type + Used bool // is the descriptor in use + ID uint32 // a unique id for this data object + Groupid uint32 // object group this data object is related to + Link uint32 // special link or relation to an id or group + Fileoff int64 // offset from start of image file + Filelen int64 // length of data in file + Storelen int64 // length of data + alignment to store data in file + + Ctime int64 // image creation time + Mtime int64 // last modification time + UID int64 // system user owning the file + Gid int64 // system group owning the file + Name [DescrNameLen]byte // descriptor name (string identifier) + Extra [DescrMaxPrivLen]byte // big enough for extra data below +} + +// Deffile represents the SIF definition-file data object descriptor +type Deffile struct { +} + +// Labels represents the SIF JSON-labels data object descriptor +type Labels struct { +} + +// Envvar represents the SIF envvar data object descriptor +type Envvar struct { +} + +// Partition represents the SIF partition data object descriptor +type Partition struct { + Fstype Fstype + Parttype Parttype + Arch [HdrArchLen]byte // arch the image is built for +} + +// Signature represents the SIF signature data object descriptor +type Signature struct { + Hashtype Hashtype + Entity [DescrEntityLen]byte +} + +// GenericJSON represents the SIF generic JSON meta-data data object descriptor +type GenericJSON struct { +} + +// Generic represents the SIF generic data object descriptor +type Generic struct { +} + +// Header describes a loaded SIF file +type Header struct { + Launch [HdrLaunchLen]byte // #! shell execution line + + Magic [HdrMagicLen]byte // look for "SIF_MAGIC" + Version [HdrVersionLen]byte // SIF version + Arch [HdrArchLen]byte // arch the primary partition is built for + ID uuid.UUID // image unique identifier + + Ctime int64 // image creation time + Mtime int64 // last modification time + + Dfree int64 // # of unused data object descr. + Dtotal int64 // # of total available data object descr. + Descroff int64 // bytes into file where descs start + Descrlen int64 // bytes used by all current descriptors + Dataoff int64 // bytes into file where data starts + Datalen int64 // bytes used by all data objects +} + +// +// This section describes SIF creation/loading data structures used when +// building or opening a SIF file. Transient data not found in the final +// SIF file. Those data structures are internal. +// + +// FileImage describes the representation of a SIF file in memory +type FileImage struct { + Header Header // the loaded SIF global header + Fp *os.File // file pointer of opened SIF file + Filesize int64 // file size of the opened SIF file + Filedata []byte // the content of the opened file + Amodebuf bool // access mode: mmap = false, buffered = true + Reader *bytes.Reader // reader on top of Mapdata + DescrArr []Descriptor // slice of loaded descriptors from SIF file + PrimPartID uint32 // ID of primary system partition if present +} + +// CreateInfo wraps all SIF file creation info needed +type CreateInfo struct { + Pathname string // the end result output filename + Launchstr string // the shell run command + Sifversion string // the SIF specification version used + ID uuid.UUID // image unique identifier + InputDescr []DescriptorInput // slice of input info for descriptor creation +} + +// DescriptorInput describes the common info needed to create a data object descriptor +type DescriptorInput struct { + Datatype Datatype // datatype being harvested for new descriptor + Groupid uint32 // group to be set for new descriptor + Link uint32 // link to be set for new descriptor + Size int64 // size of the data object for the new descriptor + Alignment int // Align requirement for data object + + Fname string // file containing data associated with the new descriptor + Fp *os.File // file pointer to opened 'fname' + Data []byte // loaded data from file + + Image *FileImage // loaded SIF file in memory + Descr *Descriptor // created end result descriptor + + Extra bytes.Buffer // where specific input type store their data +} diff --git a/sif/sif_src.go b/sif/sif_src.go new file mode 100644 index 0000000000..92af266472 --- /dev/null +++ b/sif/sif_src.go @@ -0,0 +1,287 @@ +// +build linux + +package sifimage + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "time" + + "github.com/containers/image/v5/internal/tmpdir" + "github.com/containers/image/v5/sif/internal" + "github.com/containers/image/v5/types" + "github.com/klauspost/pgzip" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + + imgspecs "github.com/opencontainers/image-spec/specs-go" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +type sifImageSource struct { + ref sifReference + sifimg internal.SifImage + workdir string + diffID digest.Digest + diffSize int64 + blobID digest.Digest + blobSize int64 + blobTime time.Time + blobType string + blobFile string + config []byte + configID digest.Digest + configSize int64 + manifest []byte +} + +func (s *sifImageSource) getLayerInfo(tarpath string) error { + ftar, err := os.Open(tarpath) + if err != nil { + return fmt.Errorf("error opening %q for reading: %v", tarpath, err) + } + defer ftar.Close() + + diffDigester := digest.Canonical.Digester() + s.diffSize, err = io.Copy(diffDigester.Hash(), ftar) + if err != nil { + return fmt.Errorf("error reading %q: %v", tarpath, err) + } + s.diffID = diffDigester.Digest() + + return nil +} +func (s *sifImageSource) createBlob(tarpath string) error { + s.blobFile = fmt.Sprintf("%s.%s", tarpath, "gz") + fgz, err := os.Create(s.blobFile) + if err != nil { + return errors.Wrapf(err, "creating file for compressed blob") + } + defer fgz.Close() + fileinfo, err := fgz.Stat() + if err != nil { + return fmt.Errorf("error reading modtime of %q: %v", s.blobFile, err) + } + s.blobTime = fileinfo.ModTime() + + ftar, err := os.Open(tarpath) + if err != nil { + return fmt.Errorf("error opening %q for reading: %v", tarpath, err) + } + defer ftar.Close() + + writer := pgzip.NewWriter(fgz) + defer writer.Close() + _, err = io.Copy(writer, ftar) + if err != nil { + return fmt.Errorf("error compressing %q: %v", tarpath, err) + } + + return nil +} + +func (s *sifImageSource) getBlobInfo() error { + fgz, err := os.Open(s.blobFile) + if err != nil { + return fmt.Errorf("error opening %q for reading: %v", s.blobFile, err) + } + defer fgz.Close() + + blobDigester := digest.Canonical.Digester() + s.blobSize, err = io.Copy(blobDigester.Hash(), fgz) + if err != nil { + return fmt.Errorf("error reading %q: %v", s.blobFile, err) + } + s.blobID = blobDigester.Digest() + + return nil +} + +// newImageSource returns an ImageSource for reading from an existing directory. +// newImageSource extracts SIF objects and saves them in a temp directory. +func newImageSource(ctx context.Context, sys *types.SystemContext, ref sifReference) (types.ImageSource, error) { + var imgSrc sifImageSource + + sifimg, err := internal.LoadSIFImage(ref.resolvedFile) + if err != nil { + return nil, errors.Wrap(err, "loading SIF file") + } + + workdir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "sif") + if err != nil { + return nil, errors.Wrapf(err, "creating temp directory") + } + + tarpath, err := sifimg.SquashFSToTarLayer(workdir) + if err != nil { + return nil, errors.Wrapf(err, "converting rootfs from SquashFS to Tarball") + } + + // generate layer info + err = imgSrc.getLayerInfo(tarpath) + if err != nil { + return nil, errors.Wrapf(err, "gathering layer diff information") + } + + // prepare compressed blob + err = imgSrc.createBlob(tarpath) + if err != nil { + return nil, errors.Wrapf(err, "creating blob file") + } + + // generate blob info + err = imgSrc.getBlobInfo() + if err != nil { + return nil, errors.Wrapf(err, "gathering blob information") + } + + // populate the rootfs section of the config + rootfs := imgspecv1.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{imgSrc.diffID}, + } + created := imgSrc.blobTime + history := []imgspecv1.History{ + { + Created: &created, + CreatedBy: fmt.Sprintf("/bin/sh -c #(nop) ADD file:%s in %c", imgSrc.diffID.Hex(), os.PathSeparator), + Comment: "imported from SIF, uuid: " + sifimg.GetSIFID(), + }, + { + Created: &created, + CreatedBy: "/bin/sh -c #(nop) CMD [\"bash\"]", + EmptyLayer: true, + }, + } + + // build an OCI image config + var config imgspecv1.Image + config.Created = &created + config.Architecture = sifimg.GetSIFArch() + config.OS = "linux" + config.RootFS = rootfs + config.History = history + err = sifimg.GetConfig(&config) + if err != nil { + return nil, errors.Wrapf(err, "getting config elements from SIF") + } + + // Encode and digest the image configuration blob. + configBytes, err := json.Marshal(&config) + if err != nil { + return nil, fmt.Errorf("error generating configuration blob for %q: %v", ref.resolvedFile, err) + } + configID := digest.Canonical.FromBytes(configBytes) + configSize := int64(len(configBytes)) + + // Populate a manifest with the configuration blob and the SquashFS part as the single layer. + layerDescriptor := imgspecv1.Descriptor{ + Digest: imgSrc.blobID, + Size: imgSrc.blobSize, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + } + manifest := imgspecv1.Manifest{ + Versioned: imgspecs.Versioned{ + SchemaVersion: 2, + }, + Config: imgspecv1.Descriptor{ + Digest: configID, + Size: configSize, + MediaType: imgspecv1.MediaTypeImageConfig, + }, + Layers: []imgspecv1.Descriptor{layerDescriptor}, + } + manifestBytes, err := json.Marshal(&manifest) + if err != nil { + return nil, fmt.Errorf("error generating manifest for %q: %v", ref.resolvedFile, err) + } + + return &sifImageSource{ + ref: ref, + sifimg: sifimg, + workdir: workdir, + diffID: imgSrc.diffID, + diffSize: imgSrc.diffSize, + blobID: imgSrc.blobID, + blobSize: imgSrc.blobSize, + blobType: layerDescriptor.MediaType, + blobFile: imgSrc.blobFile, + config: configBytes, + configID: configID, + configSize: configSize, + manifest: manifestBytes, + }, nil +} + +// Reference returns the reference used to set up this source. +func (s *sifImageSource) Reference() types.ImageReference { + return s.ref +} + +// Close removes resources associated with an initialized ImageSource, if any. +func (s *sifImageSource) Close() error { + os.RemoveAll(s.workdir) + return s.sifimg.UnloadSIFImage() +} + +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *sifImageSource) HasThreadSafeGetBlob() bool { + return false +} + +// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). +// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. +// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. +func (s *sifImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) { + // We should only be asked about things in the manifest. Maybe the configuration blob. + if info.Digest == s.configID { + return ioutil.NopCloser(bytes.NewBuffer(s.config)), s.configSize, nil + } + if info.Digest == s.blobID { + reader, err := os.Open(s.blobFile) + if err != nil { + return nil, -1, fmt.Errorf("error opening %q: %v", s.blobFile, err) + } + return reader, s.blobSize, nil + } + return nil, -1, fmt.Errorf("no blob with digest %q found", info.Digest.String()) +} + +// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). +// It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); +// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +func (s *sifImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { + if instanceDigest != nil { + return nil, "", fmt.Errorf("manifest lists are not supported by the sif transport") + } + return s.manifest, imgspecv1.MediaTypeImageManifest, nil +} + +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (s *sifImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + if instanceDigest != nil { + return nil, fmt.Errorf("manifest lists are not supported by the sif transport") + } + return nil, nil +} + +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *sifImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + return nil, nil +} diff --git a/sif/sif_transport.go b/sif/sif_transport.go new file mode 100644 index 0000000000..17814aa64a --- /dev/null +++ b/sif/sif_transport.go @@ -0,0 +1,121 @@ +// +build linux + +package sifimage + +import ( + "context" + "fmt" + "strings" + + "github.com/containers/image/v5/directory/explicitfilepath" + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/image" + "github.com/containers/image/v5/transports" + "github.com/containers/image/v5/types" + "github.com/pkg/errors" +) + +func init() { + transports.Register(Transport) +} + +// Transport is an ImageTransport for SIF images. +var Transport = sifTransport{} + +type sifTransport struct{} + +// sifReference is an ImageReference for SIF images. +type sifReference struct { + file string // As specified by the user. May be relative, contain symlinks, etc. + resolvedFile string // Absolute file path with no symlinks, at least at the time of its creation. Primarily used for policy namespaces. +} + +func (t sifTransport) Name() string { + return "sif" +} + +// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an ImageReference. +func (t sifTransport) ParseReference(reference string) (types.ImageReference, error) { + return NewReference(reference) +} + +// NewReference returns an image file reference for a specified path. +func NewReference(file string) (types.ImageReference, error) { + resolved, err := explicitfilepath.ResolvePathToFullyExplicit(file) + if err != nil { + return nil, err + } + return sifReference{file: file, resolvedFile: resolved}, nil +} + +// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys +// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value). +// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion. +// scope passed to this function will not be "", that value is always allowed. +func (t sifTransport) ValidatePolicyConfigurationScope(scope string) error { + return errors.New(`sif: does not support any scopes except the default "" one`) +} + +func (ref sifReference) Transport() types.ImageTransport { + return Transport +} + +// StringWithinTransport returns a string representation of the reference, which MUST be such that +// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference. +func (ref sifReference) StringWithinTransport() string { + return ref.file +} + +// DockerReference returns a Docker reference associated with this reference +func (ref sifReference) DockerReference() reference.Named { + return nil +} + +// PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup. +func (ref sifReference) PolicyConfigurationIdentity() string { + return ref.resolvedFile +} + +// PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search +// for if explicit configuration for PolicyConfigurationIdentity() is not set +func (ref sifReference) PolicyConfigurationNamespaces() []string { + res := []string{} + path := ref.resolvedFile + for { + lastSlash := strings.LastIndex(path, "/") + if lastSlash == -1 || path == "/" { + break + } + res = append(res, path) + path = path[:lastSlash] + } + return res +} + +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. +// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, +// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (ref sifReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { + src, err := newImageSource(ctx, sys, ref) + if err != nil { + return nil, err + } + return image.FromSource(ctx, sys, src) +} + +// NewImageSource returns a types.ImageSource for this reference. +// The caller must call .Close() on the returned ImageSource. +func (ref sifReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) { + return newImageSource(ctx, sys, ref) +} + +func (ref sifReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) { + return nil, fmt.Errorf(`"sif:" locations can only be read from, not written to`) +} + +// DeleteImage deletes the named image from the registry, if supported. +func (ref sifReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error { + return errors.Errorf("Deleting images not implemented for sif: images") +} diff --git a/transports/alltransports/alltransports.go b/transports/alltransports/alltransports.go index 2110a091d2..9dc9b4c28b 100644 --- a/transports/alltransports/alltransports.go +++ b/transports/alltransports/alltransports.go @@ -13,6 +13,8 @@ import ( _ "github.com/containers/image/v5/oci/layout" _ "github.com/containers/image/v5/openshift" _ "github.com/containers/image/v5/tarball" + + // The sif transport is registered by sif*.go // The ostree transport is registered by ostree*.go // The storage transport is registered by storage*.go "github.com/containers/image/v5/transports" diff --git a/transports/alltransports/sif.go b/transports/alltransports/sif.go new file mode 100644 index 0000000000..ba348f2d16 --- /dev/null +++ b/transports/alltransports/sif.go @@ -0,0 +1,8 @@ +// +build linux + +package alltransports + +import ( + // Register the sif transport + _ "github.com/containers/image/v5/sif" +) diff --git a/transports/alltransports/sif_stub.go b/transports/alltransports/sif_stub.go new file mode 100644 index 0000000000..7715f7be0b --- /dev/null +++ b/transports/alltransports/sif_stub.go @@ -0,0 +1,9 @@ +// +build !linux + +package alltransports + +import "github.com/containers/image/v5/transports" + +func init() { + transports.Register(transports.NewStubTransport("sif")) +}