Add nix-snapshotter support to the embedded containerd

.github/workflows/e2e.yaml

@@ -187,13 +187,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dtest: [autoimport, basics, bootstraptoken, cacerts, dualstack, etcd, hardened, lazypull, skew, secretsencryption, snapshotrestore, svcpoliciesandfirewall, token, upgrade]
+        dtest: [autoimport, basics, bootstraptoken, cacerts, dualstack, etcd, hardened, lazypull, nixsnapshotter, skew, secretsencryption, snapshotrestore, svcpoliciesandfirewall, token, upgrade]
         arch: [amd64, arm64]
         exclude:
           - dtest: autoimport
             arch: arm64
           - dtest: dualstack
             arch: arm64
+          - dtest: nixsnapshotter
+            arch: arm64
           - dtest: secretsencryption
             arch: arm64
           - dtest: snapshotrestore
@@ -245,6 +247,14 @@ jobs:
         docker image load -i ./dist/artifacts/k3s-image.tar
         IMAGE_TAG=$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep 'rancher/k3s')
         echo "K3S_IMAGE=$IMAGE_TAG" >> $GITHUB_ENV
+    - name: Install Nix
+      if: matrix.dtest == 'nixsnapshotter'
+      uses: DeterminateSystems/nix-installer-action@v17
+    - name: Build nix test image
+      if: matrix.dtest == 'nixsnapshotter'
+      run: |
+        nix build github:pdtpartners/nix-snapshotter#image-hello
+        cp result ./tests/docker/resources/nix-hello-image.tar
     - name: Download Go Tests
       uses: actions/download-artifact@v7
       with:

go.mod

@@ -125,6 +125,7 @@ require (
 	github.com/opencontainers/image-spec v1.1.1
 	github.com/opencontainers/selinux v1.13.1
 	github.com/otiai10/copy v1.14.1
+	github.com/pdtpartners/nix-snapshotter v0.4.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.23.2
 	github.com/prometheus/common v0.66.1

go.sum

@@ -1121,6 +1121,8 @@ github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs=
 github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
+github.com/pdtpartners/nix-snapshotter v0.4.0 h1:c9wxyxoJq/toKdUgoh704Tctl5bMgWksrP+fH1Trle4=
+github.com/pdtpartners/nix-snapshotter v0.4.0/go.mod h1:AuH0d9eY9rOXnI3MCzIp4BKoQCy0eIyB6CLQCEtki7M=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
@@ -1845,8 +1847,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY=
-gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/agent/config/config_linux.go

@@ -20,7 +20,7 @@ func applyContainerdOSSpecificConfig(nodeConfig *config.Node) error {
 	nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
 
 	// validate that the selected snapshotter supports the filesystem at the root path.
-	// for stargz, also overrides the image service endpoint path.
+	// for stargz and nix, also overrides the image service endpoint path.
 	switch nodeConfig.AgentConfig.Snapshotter {
 	case "overlayfs":
 		if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil {
@@ -38,6 +38,12 @@ func applyContainerdOSSpecificConfig(nodeConfig *config.Node) error {
 				nodeConfig.Containerd.Root)
 		}
 		nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+	case "nix":
+		if err := containerd.NixSupported(nodeConfig.Containerd.Root); err != nil {
+			return pkgerrors.WithMessagef(err, "\"nix\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
+				nodeConfig.Containerd.Root)
+		}
+		nodeConfig.AgentConfig.ImageServiceSocket = filepath.Join(nodeConfig.Containerd.State, "nix-snapshotter.sock")
 	}
 
 	return nil

pkg/agent/containerd/config_linux.go

@@ -3,8 +3,10 @@
 package containerd
 
 import (
+	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 
 	containerd "github.com/containerd/containerd/v2/client"
 	"github.com/containerd/containerd/v2/plugins/snapshots/overlay/overlayutils"
@@ -16,6 +18,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/moby/sys/userns"
+	"github.com/pdtpartners/nix-snapshotter/pkg/nix"
 	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
@@ -132,3 +135,10 @@ func FuseoverlayfsSupported(root string) error {
 func StargzSupported(root string) error {
 	return stargz.Supported(root)
 }
+
+func NixSupported(root string) error {
+	if _, err := exec.LookPath("nix-store"); err != nil {
+		return errors.New("nix-store not found in PATH: install nix (https://nixos.org/download) to use the nix snapshotter")
+	}
+	return nix.Supported(root)
+}

pkg/agent/containerd/config_windows.go

@@ -77,3 +77,7 @@ func FuseoverlayfsSupported(root string) error {
 func StargzSupported(root string) error {
 	return pkgerrors.WithMessagef(util3.ErrUnsupportedPlatform, "stargz is not supported")
 }
+
+func NixSupported(root string) error {
+	return pkgerrors.WithMessagef(util3.ErrUnsupportedPlatform, "nix is not supported")
+}

pkg/agent/templates/templates.go

@@ -82,7 +82,7 @@ state = {{ printf "%q" .NodeConfig.Containerd.State }}
 {{- if .NodeConfig.AgentConfig.Snapshotter }}
 [plugins."io.containerd.grpc.v1.cri".containerd]
   snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}"
-  disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}}
+  disable_snapshot_annotations = {{ if or (eq .NodeConfig.AgentConfig.Snapshotter "stargz") (eq .NodeConfig.AgentConfig.Snapshotter "nix") }}false{{else}}true{{end}}
   {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}}
 {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}
 {{ if .NodeConfig.AgentConfig.ImageServiceSocket }}
@@ -109,6 +109,27 @@ enable_keychain = true
 {{end}}
 {{end}}
 
+{{ if eq .NodeConfig.AgentConfig.Snapshotter "nix" }}
+{{ if .NodeConfig.AgentConfig.ImageServiceSocket }}
+[plugins."io.containerd.snapshotter.v1.nix"]
+  address = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}"
+
+[plugins."io.containerd.snapshotter.v1.nix".image_service]
+  enable = true
+  containerd_address = {{ deschemify .NodeConfig.Containerd.Address | printf "%q" }}
+
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux/amd64"
+  snapshotter = "nix"
+  differ = "walking"
+
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux/arm64"
+  snapshotter = "nix"
+  differ = "walking"
+{{end}}
+{{end}}
+
 {{- if or .NodeConfig.AgentConfig.CNIBinDir .NodeConfig.AgentConfig.CNIConfDir }}
 [plugins."io.containerd.grpc.v1.cri".cni]
   {{ if .NodeConfig.AgentConfig.CNIBinDir }}bin_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIBinDir }}{{end}}
@@ -190,7 +211,7 @@ state = {{ printf "%q" .NodeConfig.Containerd.State }}
 {{ with .NodeConfig.AgentConfig.Snapshotter }}
 [plugins.'io.containerd.cri.v1.images']
   snapshotter = "{{ . }}"
-  disable_snapshot_annotations = {{ if eq . "stargz" }}false{{else}}true{{end}}
+  disable_snapshot_annotations = {{ if or (eq . "stargz") (eq . "nix") }}false{{else}}true{{end}}
   use_local_image_pull = true
 {{ end }}
 
@@ -274,6 +295,32 @@ state = {{ printf "%q" .NodeConfig.Containerd.State }}
 {{ end }}
 {{ end }}
 {{ end }}
+
+{{ if eq .NodeConfig.AgentConfig.Snapshotter "nix" }}
+{{ with .NodeConfig.AgentConfig.ImageServiceSocket }}
+[plugins.'io.containerd.snapshotter.v1.nix']
+  address = {{ printf "%q" . }}
+
+[plugins.'io.containerd.snapshotter.v1.nix'.image_service]
+  enable = true
+  containerd_address = {{ deschemify $.NodeConfig.Containerd.Address | printf "%q" }}
+
+[[plugins.'io.containerd.transfer.v1.local'.unpack_config]]
+  platform = "linux/amd64"
+  snapshotter = "nix"
+  differ = "walking"
+
+[[plugins.'io.containerd.transfer.v1.local'.unpack_config]]
+  platform = "linux/arm64"
+  snapshotter = "nix"
+  differ = "walking"
+{{ end }}
+{{ end }}
+
+{{ if .IsRunningInUserNS }}
+[plugins.'io.containerd.nri.v1.nri']
+  disable = true
+{{ end }}
 `
 
 var HostsTomlHeader = "# File generated by " + version.Program + ". DO NOT EDIT.\n"

pkg/containerd/builtins_linux.go

@@ -33,4 +33,5 @@ import (
 	_ "github.com/containerd/fuse-overlayfs-snapshotter/v2/plugin"
 	_ "github.com/containerd/stargz-snapshotter/service/plugin"
 	_ "github.com/containerd/zfs/v2/plugin"
+	_ "github.com/pdtpartners/nix-snapshotter/pkg/plugin"
 )

pkg/containerd/utility_linux.go

@@ -3,9 +3,13 @@
 package containerd
 
 import (
+	"errors"
+	"os/exec"
+
 	"github.com/containerd/containerd/v2/plugins/snapshots/overlay/overlayutils"
 	fuseoverlayfs "github.com/containerd/fuse-overlayfs-snapshotter/v2"
 	stargz "github.com/containerd/stargz-snapshotter/service"
+	"github.com/pdtpartners/nix-snapshotter/pkg/nix"
 )
 
 func OverlaySupported(root string) error {
@@ -19,3 +23,10 @@ func FuseoverlayfsSupported(root string) error {
 func StargzSupported(root string) error {
 	return stargz.Supported(root)
 }
+
+func NixSupported(root string) error {
+	if _, err := exec.LookPath("nix-store"); err != nil {
+		return errors.New("nix-store not found in PATH: install nix (https://nixos.org/download) to use the nix snapshotter")
+	}
+	return nix.Supported(root)
+}

pkg/containerd/utility_windows.go

@@ -18,3 +18,7 @@ func FuseoverlayfsSupported(root string) error {
 func StargzSupported(root string) error {
 	return pkgerrors.WithMessagef(util2.ErrUnsupportedPlatform, "stargz is not supported")
 }
+
+func NixSupported(root string) error {
+	return pkgerrors.WithMessagef(util2.ErrUnsupportedPlatform, "nix is not supported")
+}

tests/docker/nixsnapshotter/nixsnapshotter_test.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
+var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
+var config *docker.TestConfig
+
+func Test_DockerNixSnapshotter(t *testing.T) {
+	flag.Parse()
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Nix Snapshotter Docker Test Suite")
+}
+
+var _ = Describe("Nix Snapshotter Tests", Ordered, func() {
+
+	Context("Setup Cluster", func() {
+		It("should provision servers with nix snapshotter", func() {
+			var err error
+			config, err = docker.NewTestConfig(*k3sImage)
+			Expect(err).NotTo(HaveOccurred())
+
+			// Write a container entrypoint wrapper that symlinks nix-store
+			// into the PATH before starting k3s. This is needed because the
+			// NixSupported() check calls exec.LookPath("nix-store") during
+			// startup, before we have a chance to docker exec into the container.
+			entrypoint := filepath.Join(config.TestDir, "nix-entrypoint.sh")
+			Expect(os.WriteFile(entrypoint, []byte("#!/bin/sh\nln -sf /nix/var/nix/profiles/default/bin/nix-store /usr/local/bin/nix-store\nexec /bin/k3s \"$@\"\n"), 0755)).To(Succeed())
+
+			os.Setenv("SERVER_DOCKER_ARGS", fmt.Sprintf("--restart=always -v /nix:/nix -v %s:/usr/local/bin/nix-entrypoint.sh --entrypoint /usr/local/bin/nix-entrypoint.sh", entrypoint))
+
+			config.ServerYaml = "snapshotter: nix"
+			Expect(config.ProvisionServers(1)).To(Succeed())
+
+			Eventually(func() error {
+				return tests.CheckDefaultDeployments(config.KubeconfigFile)
+			}, "180s", "5s").Should(Succeed())
+			Eventually(func() error {
+				return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())
+			}, "40s", "5s").Should(Succeed())
+		})
+	})
+
+	Context("Verify Nix Snapshotter", func() {
+		It("should run a pod using a nix-built image", func() {
+			// Copy the nix test image OCI tar into the k3s container.
+			// Built by CI via: nix build github:pdtpartners/nix-snapshotter#image-hello
+			cmd := fmt.Sprintf("docker cp ../resources/nix-hello-image.tar %s:/tmp/nix-hello-image.tar", config.Servers[0].Name)
+			_, err := tests.RunCommand(cmd)
+			Expect(err).NotTo(HaveOccurred(), "failed to copy test image into container")
+
+			// Run a pod using the nix:0 image reference prefix. This directs
+			// kubelet's PullImage through the nix-snapshotter image service,
+			// which loads the OCI tar and unpacks layers with nix-closure
+			// annotation processing. The snapshotter's Prepare() realizes
+			// the closure's nix store paths via nix-store and creates GC
+			// roots. The image's entrypoint (hello) prints and exits 0.
+			cmd = fmt.Sprintf("kubectl --kubeconfig=%s run nix-hello --image=nix:0/tmp/nix-hello-image.tar --image-pull-policy=Always --restart=Never", config.KubeconfigFile)
+			_, err = tests.RunCommand(cmd)
+			Expect(err).NotTo(HaveOccurred(), "failed to create nix-hello pod")
+
+			// Wait for the pod to complete successfully.
+			Eventually(func() (string, error) {
+				cmd := fmt.Sprintf("kubectl --kubeconfig=%s get pod nix-hello -o jsonpath='{.status.phase}'", config.KubeconfigFile)
+				return tests.RunCommand(cmd)
+			}, "60s", "5s").Should(Equal("Succeeded"))
+		})
+	})
+})
+
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
+var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("describe", docker.DescribeNodesAndPods(config))
+		AddReportEntry("docker-containers", docker.ListContainers())
+		AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...)))
+	}
+	if config != nil && (*ci || !failed) {
+		Expect(config.Cleanup()).To(Succeed())
+	}
+})
+