virtcontainers: Add support for ephemeral volumes

Ephemeral volumes should not be passed at 9pfs mounts.
They should be created inside the VM.

This patch disables ephemeral volumes from getting
mounted as 9pfs from the host and instead a corresponding
tmpfs is created inside the VM.

Fixes : #61

Signed-off-by: Harshal Patil <[email protected]>

cli/create.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 
 	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -103,7 +104,6 @@ func create(containerID, bundlePath, console, pidFilePath string, detach bool,
 	disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
 
 	var process vc.Process
-
 	switch containerType {
 	case vc.PodSandbox:
 		process, err = createSandbox(ociSpec, runtimeConfig, containerID, bundlePath, console, disableOutput)
@@ -259,6 +259,19 @@ func createContainer(ociSpec oci.CompatOCISpec, containerID, bundlePath,
 		return vc.Process{}, err
 	}
 
+	// Add the ephemeral device if ephemeral volume
+	// has to be attached to the container. For the given pod
+	// ephemeral volume is created only once backed by tmpfs
+	// inside the VM. For successive containers of the same
+	// pod the already existing volume is reused.
+	for _, mnt := range contConfig.Mounts {
+		if IsEphemeralStorage(mnt.Source) {
+			deviceInfo := config.DeviceInfo{}
+			deviceInfo.DevType = "e"
+			deviceInfo.ContainerPath = mnt.Source
+			contConfig.DeviceInfos = append(contConfig.DeviceInfos, deviceInfo)
+		}
+	}
 	_, c, err := vci.CreateContainer(sandboxID, contConfig)
 	if err != nil {
 		return vc.Process{}, err

cli/utils.go

@@ -43,6 +43,26 @@ func getFileContents(file string) (string, error) {
 	return string(bytes), nil
 }
 
+// IsEphemeralStorage returns true if the given path
+// to the storage belongs to kubernetes ephemeral storage
+//
+// This method depends on a specific path used by k8s
+// to detect if it's of type ephemeral. As of now,
+// this is a very k8s specific solution that works
+// but in future there should be a better way for this
+// method to determine if the path is for ephemeral
+// volume type
+func IsEphemeralStorage(path string) bool {
+	splitSourceSlice := strings.Split(path, "/")
+	if len(splitSourceSlice) > 1 {
+		storageType := splitSourceSlice[len(splitSourceSlice)-2]
+		if storageType == "kubernetes.io~empty-dir" {
+			return true
+		}
+	}
+	return false
+}
+
 func getKernelVersion() (string, error) {
 	contents, err := getFileContents(procVersion)
 	if err != nil {

cli/utils_test.go

@@ -38,6 +38,20 @@ func TestFileExists(t *testing.T) {
 		fmt.Sprintf("File %q should exist", file))
 }
 
+func TestIsEphemeralStroage(t *testing.T) {
+	sampleEphePath := "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume"
+	isEphe := IsEphemeralStorage(sampleEphePath)
+	if !isEphe {
+		t.Fatalf("Unable to correctly determine volume type")
+	}
+
+	sampleEphePath = "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/cache-volume"
+	isEphe = IsEphemeralStorage(sampleEphePath)
+	if isEphe {
+		t.Fatalf("Unable to correctly determine volume type")
+	}
+}
+
 func TestGetFileContents(t *testing.T) {
 	type testData struct {
 		contents string

virtcontainers/container.go

@@ -418,6 +418,7 @@ func (c *Container) createContainersDirs() error {
 // available when we will need to unmount those mounts.
 func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ([]Mount, error) {
 	var sharedDirMounts []Mount
+MntLoop:
 	for idx, m := range c.mounts {
 		if isSystemMount(m.Destination) || m.Type != "bind" {
 			continue
@@ -435,6 +436,16 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
 			return nil, err
 		}
 
+		for _, devInfo := range c.config.DeviceInfos {
+			if devInfo.DevType == "e" && devInfo.ContainerPath == m.Source {
+				// We are not going to bind mount host ephemeral directory
+				// provided by k8s, instead at later stage we are going to
+				// create a tmpfs inside a VM and use it to bind mount to the
+				// containers of the pod.
+				continue MntLoop
+			}
+		}
+
 		// Check if mount is a block device file. If it is, the block device will be attached to the host
 		// instead of passing this as a shared mount.
 		if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
@@ -476,7 +487,6 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
 		if err := bindMount(m.Source, mountDest, false); err != nil {
 			return nil, err
 		}
-
 		// Save HostPath mount value into the mount list of the container.
 		c.mounts[idx].HostPath = mountDest
 

virtcontainers/device/config/config.go

@@ -60,6 +60,8 @@ type DeviceInfo struct {
 	// p - FIFO
 	// b - block(buffered) special file
 	// More info in mknod(1).
+	// also,
+	// e - ephemeral volume
 	DevType string
 
 	// Major, minor numbers for device.

virtcontainers/filesystem.go

@@ -319,7 +319,6 @@ func (fs *filesystem) fetchDeviceFile(fileData []byte, devices *[]api.Device) er
 			}
 			tempDevices = append(tempDevices, &device)
 			l.Infof("Generic device unmarshalled [%v]", device)
-
 		default:
 			return fmt.Errorf("Unknown device type, could not unmarshal")
 		}

virtcontainers/kata_agent.go

@@ -19,6 +19,7 @@ import (
 	kataclient "github.com/kata-containers/agent/protocols/client"
 	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
 	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
@@ -51,6 +52,7 @@ var (
 	sharedDir9pOptions    = []string{"trans=virtio,version=9p2000.L", "nodev"}
 	shmDir                = "shm"
 	kataEphemeralDevType  = "ephemeral"
+	ephemeralPath         = filepath.Join(kataGuestSandboxDir, kataEphemeralDevType)
 )
 
 // KataAgentConfig is a structure storing information needed
@@ -775,12 +777,40 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
 		return nil, err
 	}
 
+	var devInfos []config.DeviceInfo
+	for _, devInfo := range c.config.DeviceInfos {
+		if devInfo.DevType == "e" {
+			filename := filepath.Join(ephemeralPath, filepath.Base(devInfo.ContainerPath))
+			epheStorage := &grpc.Storage{
+				Driver:     kataEphemeralDevType,
+				Source:     "tmpfs",
+				Fstype:     "tmpfs",
+				MountPoint: filename,
+			}
+			ctrStorages = append(ctrStorages, epheStorage)
+
+		} else {
+			devInfos = append(devInfos, devInfo)
+		}
+	}
+
 	// Handle container mounts
 	newMounts, err := c.mountSharedDirMounts(kataHostSharedDir, kataGuestSharedDir)
 	if err != nil {
 		return nil, err
 	}
 
+	// Modify the mount source for ephemeral volume
+	for idx, mnt := range ociSpec.Mounts {
+		for _, devInfo := range c.config.DeviceInfos {
+			if devInfo.DevType == "e" && devInfo.ContainerPath == mnt.Source {
+				ociSpec.Mounts[idx].Source = filepath.Join(ephemeralPath, filepath.Base(mnt.Source))
+			}
+		}
+	}
+
+	c.config.DeviceInfos = devInfos
+
 	// We replace all OCI mount sources that match our container mount
 	// with the right source path (The guest one).
 	if err = k.replaceOCIMountSource(ociSpec, newMounts); err != nil {