diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f69c70eb4537..f171452fa2dc 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -187,13 +187,15 @@ jobs: strategy: fail-fast: false matrix: - dtest: [autoimport, basics, bootstraptoken, cacerts, dualstack, etcd, hardened, lazypull, skew, secretsencryption, snapshotrestore, svcpoliciesandfirewall, token, upgrade] + dtest: [autoimport, basics, bootstraptoken, cacerts, dualstack, etcd, hardened, lazypull, nixsnapshotter, skew, secretsencryption, snapshotrestore, svcpoliciesandfirewall, token, upgrade] arch: [amd64, arm64] exclude: - dtest: autoimport arch: arm64 - dtest: dualstack arch: arm64 + - dtest: nixsnapshotter + arch: arm64 - dtest: secretsencryption arch: arm64 - dtest: snapshotrestore @@ -245,6 +247,14 @@ jobs: docker image load -i ./dist/artifacts/k3s-image.tar IMAGE_TAG=$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep 'rancher/k3s') echo "K3S_IMAGE=$IMAGE_TAG" >> $GITHUB_ENV + - name: Install Nix + if: matrix.dtest == 'nixsnapshotter' + uses: DeterminateSystems/nix-installer-action@v17 + - name: Build nix test image + if: matrix.dtest == 'nixsnapshotter' + run: | + nix build github:pdtpartners/nix-snapshotter#image-hello + cp result ./tests/docker/resources/nix-hello-image.tar - name: Download Go Tests uses: actions/download-artifact@v7 with: diff --git a/go.mod b/go.mod index a6be825121cc..98b3c394d3bb 100644 --- a/go.mod +++ b/go.mod @@ -125,6 +125,7 @@ require ( github.com/opencontainers/image-spec v1.1.1 github.com/opencontainers/selinux v1.13.1 github.com/otiai10/copy v1.14.1 + github.com/pdtpartners/nix-snapshotter v0.4.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/common v0.66.1 diff --git a/go.sum b/go.sum index b2cb02ab692d..0f1827701f7d 100644 --- a/go.sum +++ b/go.sum @@ -1121,6 +1121,8 @@ github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pdtpartners/nix-snapshotter v0.4.0 h1:c9wxyxoJq/toKdUgoh704Tctl5bMgWksrP+fH1Trle4= +github.com/pdtpartners/nix-snapshotter v0.4.0/go.mod h1:AuH0d9eY9rOXnI3MCzIp4BKoQCy0eIyB6CLQCEtki7M= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -1845,8 +1847,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= -gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/agent/config/config_linux.go b/pkg/agent/config/config_linux.go index 4f4a227dd18a..cb13dcb85b22 100644 --- a/pkg/agent/config/config_linux.go +++ b/pkg/agent/config/config_linux.go @@ -20,7 +20,7 @@ func applyContainerdOSSpecificConfig(nodeConfig *config.Node) error { nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock") // validate that the selected snapshotter supports the filesystem at the root path. - // for stargz, also overrides the image service endpoint path. + // for stargz and nix, also overrides the image service endpoint path. switch nodeConfig.AgentConfig.Snapshotter { case "overlayfs": if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil { @@ -38,6 +38,12 @@ func applyContainerdOSSpecificConfig(nodeConfig *config.Node) error { nodeConfig.Containerd.Root) } nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" + case "nix": + if err := containerd.NixSupported(nodeConfig.Containerd.Root); err != nil { + return pkgerrors.WithMessagef(err, "\"nix\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"", + nodeConfig.Containerd.Root) + } + nodeConfig.AgentConfig.ImageServiceSocket = filepath.Join(nodeConfig.Containerd.State, "nix-snapshotter.sock") } return nil diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go index 7a34ad7bb3bd..3c2141b9bd6a 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -3,8 +3,10 @@ package containerd import ( + "errors" "fmt" "os" + "os/exec" containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/plugins/snapshots/overlay/overlayutils" @@ -16,6 +18,7 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/version" "github.com/moby/sys/userns" + "github.com/pdtpartners/nix-snapshotter/pkg/nix" pkgerrors "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -132,3 +135,10 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return stargz.Supported(root) } + +func NixSupported(root string) error { + if _, err := exec.LookPath("nix-store"); err != nil { + return errors.New("nix-store not found in PATH: install nix (https://nixos.org/download) to use the nix snapshotter") + } + return nix.Supported(root) +} diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go index 6e5ebcdf6ac8..c915a1ef0226 100644 --- a/pkg/agent/containerd/config_windows.go +++ b/pkg/agent/containerd/config_windows.go @@ -77,3 +77,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return pkgerrors.WithMessagef(util3.ErrUnsupportedPlatform, "stargz is not supported") } + +func NixSupported(root string) error { + return pkgerrors.WithMessagef(util3.ErrUnsupportedPlatform, "nix is not supported") +} diff --git a/pkg/agent/templates/templates.go b/pkg/agent/templates/templates.go index aadf6c0112eb..b57bbfc4108d 100644 --- a/pkg/agent/templates/templates.go +++ b/pkg/agent/templates/templates.go @@ -82,7 +82,7 @@ state = {{ printf "%q" .NodeConfig.Containerd.State }} {{- if .NodeConfig.AgentConfig.Snapshotter }} [plugins."io.containerd.grpc.v1.cri".containerd] snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}" - disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}} + disable_snapshot_annotations = {{ if or (eq .NodeConfig.AgentConfig.Snapshotter "stargz") (eq .NodeConfig.AgentConfig.Snapshotter "nix") }}false{{else}}true{{end}} {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}} {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} {{ if .NodeConfig.AgentConfig.ImageServiceSocket }} @@ -109,6 +109,27 @@ enable_keychain = true {{end}} {{end}} +{{ if eq .NodeConfig.AgentConfig.Snapshotter "nix" }} +{{ if .NodeConfig.AgentConfig.ImageServiceSocket }} +[plugins."io.containerd.snapshotter.v1.nix"] + address = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}" + +[plugins."io.containerd.snapshotter.v1.nix".image_service] + enable = true + containerd_address = {{ deschemify .NodeConfig.Containerd.Address | printf "%q" }} + +[[plugins."io.containerd.transfer.v1.local".unpack_config]] + platform = "linux/amd64" + snapshotter = "nix" + differ = "walking" + +[[plugins."io.containerd.transfer.v1.local".unpack_config]] + platform = "linux/arm64" + snapshotter = "nix" + differ = "walking" +{{end}} +{{end}} + {{- if or .NodeConfig.AgentConfig.CNIBinDir .NodeConfig.AgentConfig.CNIConfDir }} [plugins."io.containerd.grpc.v1.cri".cni] {{ if .NodeConfig.AgentConfig.CNIBinDir }}bin_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIBinDir }}{{end}} @@ -190,7 +211,7 @@ state = {{ printf "%q" .NodeConfig.Containerd.State }} {{ with .NodeConfig.AgentConfig.Snapshotter }} [plugins.'io.containerd.cri.v1.images'] snapshotter = "{{ . }}" - disable_snapshot_annotations = {{ if eq . "stargz" }}false{{else}}true{{end}} + disable_snapshot_annotations = {{ if or (eq . "stargz") (eq . "nix") }}false{{else}}true{{end}} use_local_image_pull = true {{ end }} @@ -274,6 +295,32 @@ state = {{ printf "%q" .NodeConfig.Containerd.State }} {{ end }} {{ end }} {{ end }} + +{{ if eq .NodeConfig.AgentConfig.Snapshotter "nix" }} +{{ with .NodeConfig.AgentConfig.ImageServiceSocket }} +[plugins.'io.containerd.snapshotter.v1.nix'] + address = {{ printf "%q" . }} + +[plugins.'io.containerd.snapshotter.v1.nix'.image_service] + enable = true + containerd_address = {{ deschemify $.NodeConfig.Containerd.Address | printf "%q" }} + +[[plugins.'io.containerd.transfer.v1.local'.unpack_config]] + platform = "linux/amd64" + snapshotter = "nix" + differ = "walking" + +[[plugins.'io.containerd.transfer.v1.local'.unpack_config]] + platform = "linux/arm64" + snapshotter = "nix" + differ = "walking" +{{ end }} +{{ end }} + +{{ if .IsRunningInUserNS }} +[plugins.'io.containerd.nri.v1.nri'] + disable = true +{{ end }} ` var HostsTomlHeader = "# File generated by " + version.Program + ". DO NOT EDIT.\n" diff --git a/pkg/containerd/builtins_linux.go b/pkg/containerd/builtins_linux.go index c238899a884a..6054ea66de06 100644 --- a/pkg/containerd/builtins_linux.go +++ b/pkg/containerd/builtins_linux.go @@ -33,4 +33,5 @@ import ( _ "github.com/containerd/fuse-overlayfs-snapshotter/v2/plugin" _ "github.com/containerd/stargz-snapshotter/service/plugin" _ "github.com/containerd/zfs/v2/plugin" + _ "github.com/pdtpartners/nix-snapshotter/pkg/plugin" ) diff --git a/pkg/containerd/utility_linux.go b/pkg/containerd/utility_linux.go index 18eadecf24e3..925aecbbdbcc 100644 --- a/pkg/containerd/utility_linux.go +++ b/pkg/containerd/utility_linux.go @@ -3,9 +3,13 @@ package containerd import ( + "errors" + "os/exec" + "github.com/containerd/containerd/v2/plugins/snapshots/overlay/overlayutils" fuseoverlayfs "github.com/containerd/fuse-overlayfs-snapshotter/v2" stargz "github.com/containerd/stargz-snapshotter/service" + "github.com/pdtpartners/nix-snapshotter/pkg/nix" ) func OverlaySupported(root string) error { @@ -19,3 +23,10 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return stargz.Supported(root) } + +func NixSupported(root string) error { + if _, err := exec.LookPath("nix-store"); err != nil { + return errors.New("nix-store not found in PATH: install nix (https://nixos.org/download) to use the nix snapshotter") + } + return nix.Supported(root) +} diff --git a/pkg/containerd/utility_windows.go b/pkg/containerd/utility_windows.go index 3c26e2295a00..f7220a83dc8a 100644 --- a/pkg/containerd/utility_windows.go +++ b/pkg/containerd/utility_windows.go @@ -18,3 +18,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return pkgerrors.WithMessagef(util2.ErrUnsupportedPlatform, "stargz is not supported") } + +func NixSupported(root string) error { + return pkgerrors.WithMessagef(util2.ErrUnsupportedPlatform, "nix is not supported") +} diff --git a/tests/docker/nixsnapshotter/nixsnapshotter_test.go b/tests/docker/nixsnapshotter/nixsnapshotter_test.go new file mode 100644 index 000000000000..f2f0cf197248 --- /dev/null +++ b/tests/docker/nixsnapshotter/nixsnapshotter_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/k3s-io/k3s/tests" + "github.com/k3s-io/k3s/tests/docker" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers") +var ci = flag.Bool("ci", false, "running on CI, forced cleanup") +var config *docker.TestConfig + +func Test_DockerNixSnapshotter(t *testing.T) { + flag.Parse() + RegisterFailHandler(Fail) + RunSpecs(t, "Nix Snapshotter Docker Test Suite") +} + +var _ = Describe("Nix Snapshotter Tests", Ordered, func() { + + Context("Setup Cluster", func() { + It("should provision servers with nix snapshotter", func() { + var err error + config, err = docker.NewTestConfig(*k3sImage) + Expect(err).NotTo(HaveOccurred()) + + // Write a container entrypoint wrapper that symlinks nix-store + // into the PATH before starting k3s. This is needed because the + // NixSupported() check calls exec.LookPath("nix-store") during + // startup, before we have a chance to docker exec into the container. + entrypoint := filepath.Join(config.TestDir, "nix-entrypoint.sh") + Expect(os.WriteFile(entrypoint, []byte("#!/bin/sh\nln -sf /nix/var/nix/profiles/default/bin/nix-store /usr/local/bin/nix-store\nexec /bin/k3s \"$@\"\n"), 0755)).To(Succeed()) + + os.Setenv("SERVER_DOCKER_ARGS", fmt.Sprintf("--restart=always -v /nix:/nix -v %s:/usr/local/bin/nix-entrypoint.sh --entrypoint /usr/local/bin/nix-entrypoint.sh", entrypoint)) + + config.ServerYaml = "snapshotter: nix" + Expect(config.ProvisionServers(1)).To(Succeed()) + + Eventually(func() error { + return tests.CheckDefaultDeployments(config.KubeconfigFile) + }, "180s", "5s").Should(Succeed()) + Eventually(func() error { + return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames()) + }, "40s", "5s").Should(Succeed()) + }) + }) + + Context("Verify Nix Snapshotter", func() { + It("should run a pod using a nix-built image", func() { + // Copy the nix test image OCI tar into the k3s container. + // Built by CI via: nix build github:pdtpartners/nix-snapshotter#image-hello + cmd := fmt.Sprintf("docker cp ../resources/nix-hello-image.tar %s:/tmp/nix-hello-image.tar", config.Servers[0].Name) + _, err := tests.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred(), "failed to copy test image into container") + + // Run a pod using the nix:0 image reference prefix. This directs + // kubelet's PullImage through the nix-snapshotter image service, + // which loads the OCI tar and unpacks layers with nix-closure + // annotation processing. The snapshotter's Prepare() realizes + // the closure's nix store paths via nix-store and creates GC + // roots. The image's entrypoint (hello) prints and exits 0. + cmd = fmt.Sprintf("kubectl --kubeconfig=%s run nix-hello --image=nix:0/tmp/nix-hello-image.tar --image-pull-policy=Always --restart=Never", config.KubeconfigFile) + _, err = tests.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred(), "failed to create nix-hello pod") + + // Wait for the pod to complete successfully. + Eventually(func() (string, error) { + cmd := fmt.Sprintf("kubectl --kubeconfig=%s get pod nix-hello -o jsonpath='{.status.phase}'", config.KubeconfigFile) + return tests.RunCommand(cmd) + }, "60s", "5s").Should(Equal("Succeeded")) + }) + }) +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + +var _ = AfterSuite(func() { + if failed { + AddReportEntry("describe", docker.DescribeNodesAndPods(config)) + AddReportEntry("docker-containers", docker.ListContainers()) + AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...))) + } + if config != nil && (*ci || !failed) { + Expect(config.Cleanup()).To(Succeed()) + } +}) +