Initial sif transport implementation

go.mod

@@ -16,7 +16,6 @@ require (
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/mux v1.7.4 // indirect
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/imdario/mergo v0.3.12
@@ -32,13 +31,13 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
+	github.com/sylabs/sif/v2 v2.3.0
 	github.com/ulikunitz/xz v0.5.10
 	github.com/vbatts/tar-split v0.11.2
 	github.com/vbauerster/mpb/v7 v7.3.0
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0
 	go.etcd.io/bbolt v1.3.6
-	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c

sif/load.go

@@ -0,0 +1,211 @@
+package sif
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"github.com/sylabs/sif/v2/pkg/sif"
+)
+
+// injectedScriptTargetPath is the path injectedScript should be written to in the created image.
+const injectedScriptTargetPath = "/podman/runscript"
+
+// parseDefFile parses a SIF definition file from reader,
+// and returns non-trivial contents of the %environment and %runscript sections.
+func parseDefFile(reader io.Reader) ([]string, []string, error) {
+	type parserState int
+	const (
+		parsingOther parserState = iota
+		parsingEnvironment
+		parsingRunscript
+	)
+
+	environment := []string{}
+	runscript := []string{}
+
+	state := parsingOther
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		s := strings.TrimSpace(scanner.Text())
+		switch {
+		case s == `%environment`:
+			state = parsingEnvironment
+		case s == `%runscript`:
+			state = parsingRunscript
+		case strings.HasPrefix(s, "%"):
+			state = parsingOther
+		case state == parsingEnvironment:
+			if s != "" && !strings.HasPrefix(s, "#") {
+				environment = append(environment, s)
+			}
+		case state == parsingRunscript:
+			runscript = append(runscript, s)
+		default: // parsingOther: ignore the line
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, nil, fmt.Errorf("reading lines from SIF definition file object: %w", err)
+	}
+	return environment, runscript, nil
+}
+
+// generateInjectedScript generates a shell script based on
+// SIF definition file %environment and %runscript data, and returns it.
+func generateInjectedScript(environment []string, runscript []string) []byte {
+	script := fmt.Sprintf("#!/bin/bash\n"+
+		"%s\n"+
+		"%s\n", strings.Join(environment, "\n"), strings.Join(runscript, "\n"))
+	return []byte(script)
+}
+
+// processDefFile finds sif.DataDeffile in sifImage, if any,
+// and returns:
+// - the command to run
+// - contents of a script to inject as injectedScriptTargetPath, or nil
+func processDefFile(sifImage *sif.FileImage) (string, []byte, error) {
+	var environment, runscript []string
+
+	desc, err := sifImage.GetDescriptor(sif.WithDataType(sif.DataDeffile))
+	if err == nil {
+		environment, runscript, err = parseDefFile(desc.GetReader())
+		if err != nil {
+			return "", nil, err
+		}
+	}
+
+	var command string
+	var injectedScript []byte
+	if len(environment) == 0 && len(runscript) == 0 {
+		command = "bash"
+		injectedScript = nil
+	} else {
+		injectedScript = generateInjectedScript(environment, runscript)
+		command = injectedScriptTargetPath
+	}
+
+	return command, injectedScript, nil
+}
+
+func writeInjectedScript(extractedRootPath string, injectedScript []byte) error {
+	if injectedScript == nil {
+		return nil
+	}
+	filePath := filepath.Join(extractedRootPath, injectedScriptTargetPath)
+	parentDirPath := filepath.Dir(filePath)
+	if err := os.MkdirAll(parentDirPath, 0755); err != nil {
+		return fmt.Errorf("creating %s: %w", parentDirPath, err)
+	}
+	if err := ioutil.WriteFile(filePath, injectedScript, 0755); err != nil {
+		return fmt.Errorf("writing %s to %s: %w", injectedScriptTargetPath, filePath, err)
+	}
+	return nil
+}
+
+// createTarFromSIFInputs creates a tar file at tarPath, using a squashfs image at squashFSPath.
+// It can also use extractedRootPath and scriptPath, which are allocated for its exclusive use,
+// if necessary.
+func createTarFromSIFInputs(ctx context.Context, tarPath, squashFSPath string, injectedScript []byte, extractedRootPath, scriptPath string) error {
+	// It's safe for the Remove calls to happen even before we create the files, because tempDir is exclusive
+	// for our use.
+	defer os.RemoveAll(extractedRootPath)
+
+	// Almost everything in extractedRootPath comes from squashFSPath.
+	conversionCommand := fmt.Sprintf("unsquashfs -d %s -f %s && tar --acls --xattrs -C %s -cpf %s ./",
+		extractedRootPath, squashFSPath, extractedRootPath, tarPath)
+	script := "#!/bin/sh\n" + conversionCommand + "\n"
+	if err := ioutil.WriteFile(scriptPath, []byte(script), 0755); err != nil {
+		return err
+	}
+	defer os.Remove(scriptPath)
+
+	// On top of squashFSPath, we only add injectedScript, if necessary.
+	if err := writeInjectedScript(extractedRootPath, injectedScript); err != nil {
+		return err
+	}
+
+	logrus.Debugf("Converting squashfs to tar, command: %s ...", conversionCommand)
+	cmd := exec.CommandContext(ctx, "fakeroot", "--", scriptPath)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("converting image: %w, output: %s", err, string(output))
+	}
+	logrus.Debugf("... finished converting squashfs to tar")
+	return nil
+}
+
+// convertSIFToElements processes sifImage and creates/returns
+// the relevant elements for contructing an OCI-like image:
+// - A path to a tar file containing a root filesystem,
+// - A command to run.
+// The returned tar file path is inside tempDir, which can be assumed to be empty
+// at start, and is exclusively used by the current process (i.e. it is safe
+// to use hard-coded relative paths within it).
+func convertSIFToElements(ctx context.Context, sifImage *sif.FileImage, tempDir string) (string, []string, error) {
+	// We could allocate unique names for all of these using ioutil.Temp*, but tempDir is exclusive,
+	// so we can just hard-code a set of unique values here.
+	// We create and/or manage cleanup of these two paths.
+	squashFSPath := filepath.Join(tempDir, "rootfs.squashfs")
+	tarPath := filepath.Join(tempDir, "rootfs.tar")
+	// We only allocate these paths, the user is responsible for cleaning them up.
+	extractedRootPath := filepath.Join(tempDir, "rootfs")
+	scriptPath := filepath.Join(tempDir, "script")
+
+	succeeded := false
+	// It's safe for the Remove calls to happen even before we create the files, because tempDir is exclusive
+	// for our use.
+	// Ideally we would remove squashFSPath immediately after creating extractedRootPath, but we need
+	// to run both creation and consumption of extractedRootPath in the same fakeroot context.
+	// So, overall, this process requires at least 2 compressed copies (SIF and squashFSPath) and 2
+	// uncompressed copies (extractedRootPath and tarPath) of the data, all using up space at the same time.
+	// That's rather unsatisfactory, ideally we would be streaming the data directly from a squashfs parser
+	// reading from the SIF file to a tarball, for 1 compresed and 1 uncompressed copy.
+	defer os.Remove(squashFSPath)
+	defer func() {
+		if !succeeded {
+			os.Remove(tarPath)
+		}
+	}()
+
+	command, injectedScript, err := processDefFile(sifImage)
+	if err != nil {
+		return "", nil, err
+	}
+
+	rootFS, err := sifImage.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys))
+	if err != nil {
+		return "", nil, fmt.Errorf("looking up rootfs from SIF file: %w", err)
+	}
+	// TODO: We'd prefer not to make a full copy of the file here; unsquashfs ≥ 4.4
+	// has an -o option that allows extracting a squashfs from the SIF file directly,
+	// but that version is not currently available in RHEL 8.
+	logrus.Debugf("Creating a temporary squashfs image %s ...", squashFSPath)
+	if err := func() error { // A scope for defer
+		f, err := os.Create(squashFSPath)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
+		if _, err := io.CopyN(f, rootFS.GetReader(), rootFS.Size()); err != nil {
+			return err
+		}
+		return nil
+	}(); err != nil {
+		return "", nil, err
+	}
+	logrus.Debugf("... finished creating a temporary squashfs image")
+
+	if err := createTarFromSIFInputs(ctx, tarPath, squashFSPath, injectedScript, extractedRootPath, scriptPath); err != nil {
+		return "", nil, err
+	}
+	succeeded = true
+	return tarPath, []string{command}, nil
+}

sif/load_test.go

@@ -0,0 +1,58 @@
+package sif
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseDefFile(t *testing.T) {
+	for _, c := range []struct {
+		name        string
+		input       string
+		environment []string
+		runscript   []string
+	}{
+		{"Empty input", "", []string{}, []string{}},
+		{
+			name: "Basic smoke test",
+			input: "Bootstrap: library\n" +
+				"%environment\n" +
+				"			export FOO=world\n" +
+				"			export BAR=baz\n" +
+				"%runscript\n" +
+				`			echo "Hello $FOO"` + "\n" +
+				"			sleep 5\n" +
+				"%help\n" +
+				"			Abandon all hope.\n",
+			environment: []string{"export FOO=world", "export BAR=baz"},
+			runscript:   []string{`echo "Hello $FOO"`, "sleep 5"},
+		},
+		{
+			name: "Trailing section marker",
+			input: "Bootstrap: library\n" +
+				"%environment\n" +
+				"			export FOO=world\n" +
+				"%runscript",
+			environment: []string{"export FOO=world"},
+			runscript:   []string{},
+		},
+	} {
+		env, rs, err := parseDefFile(bytes.NewReader([]byte(c.input)))
+		require.NoError(t, err, c.name)
+		assert.Equal(t, c.environment, env, c.name)
+		assert.Equal(t, c.runscript, rs, c.name)
+	}
+}
+
+func TestGenerateInjectedScript(t *testing.T) {
+	res := generateInjectedScript([]string{"export FOO=world", "export BAR=baz"},
+		[]string{`echo "Hello $FOO"`, "sleep 5"})
+	assert.Equal(t, "#!/bin/bash\n"+
+		"export FOO=world\n"+
+		"export BAR=baz\n"+
+		`echo "Hello $FOO"`+"\n"+
+		"sleep 5\n", string(res))
+}