ociruntime: handle images with high layer count

When the action required an image with more than 20 layers, our mount
will fail with

```
create OCI bundle: create rootfs: mount overlayfs: no such file or directory
```

After some digging, it seems like 20 is the current limit of the number
of lowerdir allowed in each mount call.

Add special logic to break down images with more than 20 layers into
groups of 20. For each group, create an overlayfs mount called
"merged<group-id>" in the same bundle dir. The final overlayfs will then
be composed of these "merged" groups as lowerdirs.

enterprise/server/remote_execution/containers/ociruntime/ociruntime.go

@@ -210,6 +210,7 @@ type ociContainer struct {
 
 	cid              string
 	workDir          string
+	mergedMounts     []string
 	overlayfsMounted bool
 	stats            container.UsageStats
 	networkPool      *networking.ContainerNetworkPool
@@ -441,6 +442,14 @@ func (c *ociContainer) Remove(ctx context.Context) error {
 		firstErr = status.UnavailableErrorf("delete container: %s", err)
 	}
 
+	if len(c.mergedMounts) > 0 {
+		for _, merged := range c.mergedMounts {
+			if err := syscall.Unmount(merged, syscall.MNT_FORCE); err != nil && firstErr == nil {
+				firstErr = status.UnavailableErrorf("unmount overlayfs: %s", err)
+			}
+		}
+	}
+
 	if c.overlayfsMounted {
 		if err := syscall.Unmount(c.rootfsPath(), syscall.MNT_FORCE); err != nil && firstErr == nil {
 			firstErr = status.UnavailableErrorf("unmount overlayfs: %s", err)
@@ -546,16 +555,13 @@ func (c *ociContainer) createRootfs(ctx context.Context) error {
 	}
 
 	// Create an overlayfs with the pulled image layers.
-	var lowerDirs []string
 	image, ok := c.imageStore.CachedImage(c.imageRef)
 	if !ok {
 		return fmt.Errorf("bad state: attempted to create rootfs before pulling image")
 	}
-	// overlayfs "lowerdir" mount args are ordered from uppermost to lowermost,
-	// but manifest layers are ordered from lowermost to uppermost. So we
-	// iterate in reverse order when building the lowerdir args.
-	for i := len(image.Layers) - 1; i >= 0; i-- {
-		layer := image.Layers[i]
+	mergedCount := 0
+	var lowerDirs, mergedDirs []string
+	for i, layer := range image.Layers {
 		path := layerPath(c.imageCacheRoot, layer.DiffID)
 		// Skip empty dirs - these can cause conflicts since they will always
 		// have the same digest, and also just add more overhead.
@@ -568,6 +574,42 @@ func (c *ociContainer) createRootfs(ctx context.Context) error {
 			continue
 		}
 		lowerDirs = append(lowerDirs, path)
+
+		// Overlayfs mount has a limit of 20 lowerdirs.
+		// Merge every 20 lower dir into a new overlayfs to avoid the limit
+		if len(lowerDirs) == 20 || i == len(image.Layers)-1 {
+			workdir := filepath.Join(c.bundlePath(), "tmp", fmt.Sprintf("merged%d.work", mergedCount))
+			if err := os.MkdirAll(workdir, 0755); err != nil {
+				return fmt.Errorf("create overlay workdir: %w", err)
+			}
+			upperdir := filepath.Join(c.bundlePath(), "tmp", fmt.Sprintf("merged%d.upper", mergedCount))
+			if err := os.MkdirAll(upperdir, 0755); err != nil {
+				return fmt.Errorf("create overlay upperdir: %w", err)
+			}
+			merged := filepath.Join(c.bundlePath(), "tmp", fmt.Sprintf("merged%d", mergedCount))
+			if err := os.MkdirAll(merged, 0755); err != nil {
+				return fmt.Errorf("create overlay merged: %w", err)
+			}
+			slices.Reverse(lowerDirs)
+			options := fmt.Sprintf(
+				"lowerdir=%s,upperdir=%s,workdir=%s,userxattr,volatile",
+				strings.Join(lowerDirs, ":"), upperdir, workdir)
+			log.CtxDebugf(ctx, "Mounting overlayfs to %q, options=%q", merged, options)
+			if err := syscall.Mount("none", merged, "overlay", 0, options); err != nil {
+				return fmt.Errorf("mount overlayfs: %w", err)
+			}
+
+			mergedDirs = append(mergedDirs, merged)
+			lowerDirs = []string{}
+			mergedCount++
+		}
+	}
+	if len(mergedDirs) != 0 {
+		c.mergedMounts = mergedDirs
+		lowerDirs = mergedDirs
+	}
+	if len(lowerDirs) > 20 {
+		log.CtxWarningf(ctx, "Image %q has too many layers (%d) for overlayfs", c.imageRef, len(lowerDirs))
 	}
 	// Create workdir and upperdir.
 	workdir := filepath.Join(c.bundlePath(), "tmp", "rootfs.work")
@@ -579,6 +621,11 @@ func (c *ociContainer) createRootfs(ctx context.Context) error {
 		return fmt.Errorf("create overlay upperdir: %w", err)
 	}
 
+	// overlayfs "lowerdir" mount args are ordered from uppermost to lowermost,
+	// but manifest layers are ordered from lowermost to uppermost. So we need to
+	// reverse the order before constructing the mount option.
+	slices.Reverse(lowerDirs)
+
 	// TODO: do this mount inside a namespace so that it gets removed even if
 	// the executor crashes (also needed for rootless support)
 

enterprise/server/remote_execution/containers/ociruntime/ociruntime_test.go

@@ -922,6 +922,41 @@ func TestOverlayfsEdgeCases(t *testing.T) {
 	assert.Equal(t, 0, res.ExitCode)
 }
 
+func TestOverlayfsHighLayerCount(t *testing.T) {
+	setupNetworking(t)
+
+	image := "ghcr.io/avdv/nix-build@sha256:5f731adacf7290352fed6c1960dfb56ec3fdb31a376d0f2170961fbc96944d50"
+
+	ctx := context.Background()
+	env := testenv.GetTestEnv(t)
+
+	runtimeRoot := testfs.MakeTempDir(t)
+	flags.Set(t, "executor.oci.runtime_root", runtimeRoot)
+
+	buildRoot := testfs.MakeTempDir(t)
+
+	provider, err := ociruntime.NewProvider(env, buildRoot)
+	require.NoError(t, err)
+	wd := testfs.MakeDirAll(t, buildRoot, "work")
+
+	c, err := provider.New(ctx, &container.Init{Props: &platform.Properties{
+		ContainerImage: image,
+	}})
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		err := c.Remove(ctx)
+		require.NoError(t, err)
+	})
+
+	// Run
+	cmd := &repb.Command{Arguments: []string{"echo", "1"}}
+	res := c.Run(ctx, cmd, wd, oci.Credentials{})
+	require.NoError(t, res.Error)
+	assert.Equal(t, "1\n", string(res.Stdout))
+	assert.Empty(t, string(res.Stderr))
+	assert.Equal(t, 0, res.ExitCode)
+}
+
 func TestPersistentWorker(t *testing.T) {
 	setupNetworking(t)