sif: initial sif transport implementation

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/opencontainers/selinux v1.9.1
 	github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913
 	github.com/pkg/errors v0.9.1
+	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	github.com/ulikunitz/xz v0.5.10

go.sum

@@ -603,6 +603,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=

sif/internal/sif_util.go

@@ -0,0 +1,238 @@
+// +build linux
+
+package internal
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/containers/image/v5/sif/sif"
+	"github.com/pkg/errors"
+
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+type SifImage struct {
+	fimg      sif.FileImage
+	rootfs    *sif.Descriptor
+	deffile   *sif.Descriptor
+	defReader *io.SectionReader
+	cmdlist   []string
+	runscript *bytes.Buffer
+	env       *sif.Descriptor
+	envReader *io.SectionReader
+	envlist   []string
+}
+
+func LoadSIFImage(path string) (image SifImage, err error) {
+	// open up the SIF file and get its header
+	image.fimg, err = sif.LoadContainer(path, true)
+	if err != nil {
+		return
+	}
+
+	// check for a system partition and save it
+	image.rootfs, _, err = image.fimg.GetPartPrimSys()
+	if err != nil {
+		return SifImage{}, errors.Wrap(err, "looking up rootfs from SIF file")
+	}
+
+	// look for a definition file object
+	searchDesc := sif.Descriptor{Datatype: sif.DataDeffile}
+	resultDescs, _, err := image.fimg.GetFromDescr(searchDesc)
+	if err == nil && resultDescs != nil {
+		// we assume in practice that typical SIF files don't hold multiple deffiles
+		image.deffile = resultDescs[0]
+		image.defReader = io.NewSectionReader(image.fimg.Fp, image.deffile.Fileoff, image.deffile.Filelen)
+	}
+	if err = image.generateConfig(); err != nil {
+		return SifImage{}, err
+	}
+
+	// look for an environment variable set object
+	searchDesc = sif.Descriptor{Datatype: sif.DataEnvVar}
+	resultDescs, _, err = image.fimg.GetFromDescr(searchDesc)
+	if err == nil && resultDescs != nil {
+		// we assume in practice that typical SIF files don't hold multiple EnvVar sets
+		image.env = resultDescs[0]
+		image.envReader = io.NewSectionReader(image.fimg.Fp, image.env.Fileoff, image.env.Filelen)
+	}
+
+	return image, nil
+}
+
+func (image *SifImage) parseEnvironment(scanner *bufio.Scanner) error {
+	for scanner.Scan() {
+		s := strings.TrimSpace(scanner.Text())
+		if s == "" || strings.HasPrefix(s, "#") {
+			continue
+		}
+		if strings.HasPrefix(s, "%") {
+			return nil
+		}
+		image.envlist = append(image.envlist, s)
+	}
+	if err := scanner.Err(); err != nil {
+		return errors.Wrap(err, "parsing environment from SIF definition file object")
+	}
+	return nil
+}
+
+func (image *SifImage) parseRunscript(scanner *bufio.Scanner) error {
+	for scanner.Scan() {
+		s := strings.TrimSpace(scanner.Text())
+		if strings.HasPrefix(s, "%") {
+			return nil
+		}
+		image.cmdlist = append(image.cmdlist, s)
+	}
+	if err := scanner.Err(); err != nil {
+		return errors.Wrap(err, "parsing runscript from SIF definition file object")
+	}
+	return nil
+}
+
+func (image *SifImage) generateRunscript() error {
+	base := `#!/bin/bash
+`
+	image.runscript = bytes.NewBufferString(base)
+	for _, s := range image.envlist {
+		_, err := image.runscript.WriteString(fmt.Sprintln(s))
+		if err != nil {
+			return errors.Wrap(err, "writing to runscript buffer")
+		}
+	}
+	for _, s := range image.cmdlist {
+		_, err := image.runscript.WriteString(fmt.Sprintln(s))
+		if err != nil {
+			return errors.Wrap(err, "writing to runscript buffer")
+		}
+	}
+	return nil
+}
+
+func (image *SifImage) generateConfig() error {
+	if image.deffile == nil {
+		image.cmdlist = append(image.cmdlist, "bash")
+		return nil
+	}
+
+	// extract %environment/%runscript from definition file
+	var err error
+	scanner := bufio.NewScanner(image.defReader)
+	for scanner.Scan() {
+		s := strings.TrimSpace(scanner.Text())
+	again:
+		if s == `%environment` {
+			if err = image.parseEnvironment(scanner); err != nil {
+				return err
+			}
+		} else if s == `%runscript` {
+			if err = image.parseRunscript(scanner); err != nil {
+				return err
+			}
+		}
+		s = strings.TrimSpace(scanner.Text())
+		if s == `%environment` || s == `%runscript` {
+			goto again
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return errors.Wrap(err, "reading lines from SIF definition file object")
+	}
+
+	if len(image.cmdlist) == 0 && len(image.envlist) == 0 {
+		image.cmdlist = append(image.cmdlist, "bash")
+	} else {
+		if err = image.generateRunscript(); err != nil {
+			return errors.Wrap(err, "generating runscript")
+		}
+		image.cmdlist = []string{"/podman/runscript"}
+	}
+
+	return nil
+}
+
+func (image SifImage) GetConfig(config *imgspecv1.Image) error {
+	config.Config.Cmd = append(config.Config.Cmd, image.cmdlist...)
+	return nil
+}
+
+func (image SifImage) UnloadSIFImage() (err error) {
+	err = image.fimg.UnloadContainer()
+	return
+}
+
+func (image SifImage) GetSIFID() string {
+	return image.fimg.Header.ID.String()
+}
+
+func (image SifImage) GetSIFArch() string {
+	return sif.GetGoArch(string(image.fimg.Header.Arch[:sif.HdrArchLen-1]))
+}
+
+const squashFilename = "rootfs.squashfs"
+const tarFilename = "rootfs.tar"
+
+func runUnSquashFSTar(tempdir string) (err error) {
+	script := `
+#!/bin/sh
+unsquashfs -f ` + squashFilename + ` && tar --acls --xattrs -C ./squashfs-root -cpf ` + tarFilename + ` ./
+`
+
+	if err = ioutil.WriteFile(filepath.Join(tempdir, "script"), []byte(script), 0755); err != nil {
+		return err
+	}
+	cmd := []string{"fakeroot", "--", "./script"}
+
+	xcmd := exec.Command(cmd[0], cmd[1:]...)
+	xcmd.Stderr = os.Stderr
+	xcmd.Dir = tempdir
+	err = xcmd.Run()
+	return
+}
+
+func (image *SifImage) writeRunscript(tempdir string) (err error) {
+	if image.runscript == nil {
+		return nil
+	}
+	rsPath := filepath.Join(tempdir, "squashfs-root", "podman")
+	if err = os.MkdirAll(rsPath, 0755); err != nil {
+		return
+	}
+	if err = ioutil.WriteFile(filepath.Join(rsPath, "runscript"), image.runscript.Bytes(), 0755); err != nil {
+		return errors.Wrap(err, "writing /podman/runscript")
+	}
+	return nil
+}
+
+func (image SifImage) SquashFSToTarLayer(tempdir string) (tarpath string, err error) {
+	if _, err = image.fimg.Fp.Seek(image.rootfs.Fileoff, 0); err != nil {
+		return
+	}
+	f, err := os.Create(filepath.Join(tempdir, squashFilename))
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	if _, err = io.CopyN(f, image.fimg.Fp, image.rootfs.Filelen); err != nil {
+		return
+	}
+	if err = f.Sync(); err != nil {
+		return
+	}
+	if err = image.writeRunscript(tempdir); err != nil {
+		return
+	}
+	if err = runUnSquashFSTar(tempdir); err != nil {
+		return
+	}
+	return filepath.Join(tempdir, tarFilename), nil
+}

sif/sif/LICENSE.md

@@ -0,0 +1,27 @@
+Copyright (c) 2018, Sylabs Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from this
+   software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.