Add support for extra-rootfs-dir

cmd/rep/config/config.go

@@ -88,6 +88,7 @@ type RepConfig struct {
 	CommunicationTimeout      durationjson.Duration `json:"communication_timeout,omitempty"`
 	EvacuationPollingInterval durationjson.Duration `json:"evacuation_polling_interval,omitempty"`
 	EvacuationTimeout         durationjson.Duration `json:"evacuation_timeout,omitempty"`
+	ExtraRootfsDir            string                `json:"extra_root_fs_dir"`
 	LayeringMode              string                `json:"layering_mode,omitempty"`
 	ListenAddr                string                `json:"listen_addr,omitempty"`
 	ListenAddrSecurable       string                `json:"listen_addr_securable,omitempty"`

cmd/rep/main.go

@@ -8,9 +8,11 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io/fs"
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -91,6 +93,28 @@ func main() {
 	rootFSMap := repConfig.PreloadedRootFS.StackPathMap()
 	sidecarRootFSPath := repConfig.SidecarRootFSPath
 
+	if sidecarRootFSPath == "" {
+		for _, rootFSPath := range rootFSMap {
+			sidecarRootFSPath = rootFSPath
+			break
+		}
+	}
+
+	walkDirErr := filepath.WalkDir(repConfig.ExtraRootfsDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		ext := filepath.Ext(path)
+		if strings.EqualFold(ext, ".tar") {
+			key := strings.TrimSuffix(filepath.Base(path), ext)
+			rootFSMap[key] = path
+		}
+		return nil
+	})
+	if walkDirErr != nil {
+		logger.Debug("missing-extra-rootfs", lager.Data{"error": walkDirErr})
+	}
+
 	executorClient, containerMetricsProvider, executorMembers, err := executorinit.Initialize(logger, repConfig.ExecutorConfig, repConfig.CellID, repConfig.Zone, rootFSMap, sidecarRootFSPath, metronClient, clock)
 	if err != nil {
 		logger.Error("failed-to-initialize-executor", err)

cmd/rep/main_test.go

@@ -10,6 +10,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
@@ -418,6 +419,45 @@ var _ = Describe("The Rep", func() {
 				),
 			))
 		})
+
+		Context("when there are extra rootfses", func() {
+			var extraRootfs, rootfsTar string
+			BeforeEach(func() {
+				var err error
+				extraRootfs, err = os.MkdirTemp("", "extra-rootfs-dir-*")
+				Expect(err).NotTo(HaveOccurred())
+				rootfsTar = filepath.Join(extraRootfs, "special-rootfs.tar")
+				file, err := os.Create(rootfsTar)
+				Expect(err).NotTo(HaveOccurred())
+				defer file.Close()
+
+				repConfig.ExtraRootfsDir = extraRootfs
+			})
+
+			AfterEach(func() {
+				Expect(os.RemoveAll(extraRootfs)).To(Succeed())
+			})
+
+			It("creates containers with the correct rootfses", func() {
+				var createRequest1, createRequest2, createRequest3 string
+				Eventually(createRequestReceived).Should(Receive(&createRequest1))
+				Eventually(createRequestReceived).Should(Receive(&createRequest2))
+				Eventually(createRequestReceived).Should(Receive(&createRequest3))
+				Expect([]string{createRequest1, createRequest2, createRequest3}).To(ConsistOf(
+					And(
+						ContainSubstring(`rootfs-c`),
+						ContainSubstring(`"image":{"uri":"/path/to/another/rootfs"}`),
+					),
+					And(
+						ContainSubstring(`rootfs-c`),
+						ContainSubstring(`"image":{"uri":"/path/to/rootfs"}`),
+					),
+					And(
+						ContainSubstring(`rootfs-c`),
+						ContainSubstring(fmt.Sprintf(`"image":{"uri":"%s"}`, rootfsTar)),
+					)))
+			})
+		})
 	})
 
 	Describe("RootFS for garden healthcheck", func() {
@@ -465,6 +505,36 @@ var _ = Describe("The Rep", func() {
 			})
 		})
 
+		Context("when there are extra rootfses", func() {
+			var extraRootfs, rootfsTar string
+			BeforeEach(func() {
+				var err error
+				extraRootfs, err = os.MkdirTemp("", "extra-rootfs-dir-*")
+				Expect(err).NotTo(HaveOccurred())
+				rootfsTar = filepath.Join(extraRootfs, "special-rootfs.tar")
+				file, err := os.Create(rootfsTar)
+				Expect(err).NotTo(HaveOccurred())
+				defer file.Close()
+
+				repConfig.ExtraRootfsDir = extraRootfs
+				repConfig.PreloadedRootFS = append([]config.RootFS{{
+					Name: "another",
+					Path: "/path/to/another/rootfs",
+				}}, repConfig.PreloadedRootFS...)
+			})
+
+			AfterEach(func() {
+				Expect(os.RemoveAll(extraRootfs)).To(Succeed())
+			})
+
+			It("uses the first rootfs", func() {
+				Eventually(createRequestReceived).Should(Receive(And(
+					ContainSubstring(`check-`),
+					ContainSubstring(`/rootfs`),
+				)))
+			})
+		})
+
 		Context("when there is an explixitly specified rootfs path", func() {
 			BeforeEach(func() {
 				repConfig.PreloadedRootFS = append(repConfig.PreloadedRootFS, config.RootFS{