Fix lxd, so that it works with lxcfs

nixos/modules/virtualisation/lxd.nix

@@ -15,7 +15,6 @@ in
   ###### interface
 
   options = {
-
     virtualisation.lxd = {
       enable = mkOption {
         type = types.bool;
@@ -25,12 +24,18 @@ in
           containers. Users in the "lxd" group can interact with
           the daemon (e.g. to start or stop containers) using the
           <command>lxc</command> command line tool, among others.
+
+          Most of the time, you'll also want to start lxcfs, so
+          that containers can "see" the limits:
+          <code>
+            virtualisation.lxc.lxcfs.enable = true;
+          </code>
         '';
       };
 
       package = mkOption {
         type = types.package;
-        default = pkgs.lxd;
+        default = pkgs.lxd.override { nftablesSupport = config.networking.nftables.enable; };
         defaultText = "pkgs.lxd";
         description = ''
           The LXD package to use.
@@ -65,6 +70,7 @@ in
           with nixos.
         '';
       };
+
       recommendedSysctlSettings = mkOption {
         type = types.bool;
         default = false;
@@ -83,7 +89,6 @@ in
   ###### implementation
 
   config = mkIf cfg.enable {
-
     environment.systemPackages = [ cfg.package ];
 
     security.apparmor = {
@@ -115,6 +120,12 @@ in
         LimitNOFILE = "1048576";
         LimitNPROC = "infinity";
         TasksMax = "infinity";
+
+        # By default, `lxd` loads configuration files from hard-coded
+        # `/usr/share/lxc/config` - since this is a no-go for us, we have to
+        # explicitly tell it where the actual configuration files are
+        Environment = mkIf (config.virtualisation.lxc.lxcfs.enable)
+          "LXD_LXC_TEMPLATE_CONFIG=${pkgs.lxcfs}/share/lxc/config";
       };
     };
 

nixos/tests/all-tests.nix

@@ -178,6 +178,8 @@ in
   limesurvey = handleTest ./limesurvey.nix {};
   login = handleTest ./login.nix {};
   loki = handleTest ./loki.nix {};
+  lxd = handleTest ./lxd.nix {};
+  lxd-nftables = handleTest ./lxd-nftables.nix {};
   #logstash = handleTest ./logstash.nix {};
   lorri = handleTest ./lorri/default.nix {};
   magnetico = handleTest ./magnetico.nix {};

nixos/tests/lxd-nftables.nix

@@ -0,0 +1,50 @@
+# This test makes sure that lxd stops implicitly depending on iptables when
+# user enabled nftables.
+#
+# It has been extracted from `lxd.nix` for clarity, and because switching from
+# iptables to nftables requires a full reboot, which is a bit hard inside NixOS
+# tests.
+
+import ./make-test-python.nix ({ pkgs, ...} : {
+  name = "lxd-nftables";
+  meta = with pkgs.stdenv.lib.maintainers; {
+    maintainers = [ patryk27 ];
+  };
+
+  machine = { lib, ... }: {
+    virtualisation = {
+      lxd.enable = true;
+    };
+
+    networking = {
+      firewall.enable = false;
+      nftables.enable = true;
+      nftables.ruleset = ''
+        table inet filter {
+          chain incoming {
+            type filter hook input priority 0;
+            policy accept;
+          }
+
+          chain forward {
+            type filter hook forward priority 0;
+            policy accept;
+          }
+
+          chain output {
+            type filter hook output priority 0;
+            policy accept;
+          }
+        }
+      '';
+    };
+  };
+
+  testScript = ''
+    machine.wait_for_unit("network.target")
+
+    with subtest("When nftables are enabled, lxd doesn't depend on iptables anymore"):
+        machine.succeed("lsmod | grep nf_tables")
+        machine.fail("lsmod | grep ip_tables")
+  '';
+})

nixos/tests/lxd.nix

@@ -0,0 +1,135 @@
+import ./make-test-python.nix ({ pkgs, ...} :
+
+let
+  # Since we don't have access to the internet during the tests, we have to
+  # pre-fetch lxd containers beforehand.
+  #
+  # I've chosen to import Alpine Linux, because its image is turbo-tiny and,
+  # generally, sufficient for our tests.
+
+  alpine-meta = pkgs.fetchurl {
+    url = "https://uk.images.linuxcontainers.org/images/alpine/3.11/i386/default/20200608_13:00/lxd.tar.xz";
+    sha256 = "1hkvaj3rr333zmx1759njy435lps33gl4ks8zfm7m4nqvipm26a0";
+  };
+
+  alpine-rootfs = pkgs.fetchurl {
+    url = "https://uk.images.linuxcontainers.org/images/alpine/3.11/i386/default/20200608_13:00/rootfs.tar.xz";
+    sha256 = "1v82zdra4j5xwsff09qlp7h5vbsg54s0j7rdg4rynichfid3r347";
+  };
+
+  lxd-config = pkgs.writeText "config.yaml" ''
+    storage_pools:
+      - name: default
+        driver: dir
+        config:
+          source: /var/lxd-pool
+
+    networks:
+      - name: lxdbr0
+        type: bridge
+        config:
+          ipv4.address: auto
+          ipv6.address: none
+
+    profiles:
+      - name: default
+        devices:
+          eth0:
+            name: eth0
+            network: lxdbr0
+            type: nic
+          root:
+            path: /
+            pool: default
+            type: disk
+  '';
+
+in {
+  name = "lxd";
+  meta = with pkgs.stdenv.lib.maintainers; {
+    maintainers = [ patryk27 ];
+  };
+
+  machine = { lib, ... }: {
+    virtualisation = {
+      # Since we're testing `limits.cpu`, we've gotta have a known number of
+      # cores to lay on
+      cores = 2;
+
+      # Ditto, for `limits.memory`
+      memorySize = 512;
+
+      lxc.lxcfs.enable = true;
+      lxd.enable = true;
+    };
+  };
+
+  testScript = ''
+    machine.wait_for_unit("sockets.target")
+    machine.wait_for_unit("lxd.service")
+
+    # It takes additional second for lxd to settle
+    machine.sleep(1)
+
+    # lxd expects the pool's directory to already exist
+    machine.succeed("mkdir /var/lxd-pool")
+
+    machine.succeed(
+        "cat ${lxd-config} | lxd init --preseed"
+    )
+
+    machine.succeed(
+        "lxc image import ${alpine-meta} ${alpine-rootfs} --alias alpine"
+    )
+
+    with subtest("Containers can be launched and destroyed"):
+        machine.succeed("lxc launch alpine test")
+        machine.succeed("lxc exec test true")
+        machine.succeed("lxc delete -f test")
+
+    with subtest("Containers are being mounted with lxcfs inside"):
+        machine.succeed("lxc launch alpine test")
+
+        ## ---------- ##
+        ## limits.cpu ##
+
+        machine.succeed("lxc config set test limits.cpu 1")
+
+        # Since Alpine doesn't have `nproc` pre-installed, we've gotta resort
+        # to the primal methods
+        assert (
+            "1"
+            == machine.succeed("lxc exec test grep -- -c ^processor /proc/cpuinfo").strip()
+        )
+
+        machine.succeed("lxc config set test limits.cpu 2")
+
+        assert (
+            "2"
+            == machine.succeed("lxc exec test grep -- -c ^processor /proc/cpuinfo").strip()
+        )
+
+        ## ------------- ##
+        ## limits.memory ##
+
+        machine.succeed("lxc config set test limits.memory 64MB")
+
+        assert (
+            "MemTotal:          62500 kB"
+            == machine.succeed("lxc exec test grep -- MemTotal /proc/meminfo").strip()
+        )
+
+        machine.succeed("lxc config set test limits.memory 128MB")
+
+        assert (
+            "MemTotal:         125000 kB"
+            == machine.succeed("lxc exec test grep -- MemTotal /proc/meminfo").strip()
+        )
+
+        machine.succeed("lxc delete -f test")
+
+    with subtest("Unless explicitly changed, lxd leans on iptables"):
+        machine.succeed("lsmod | grep ip_tables")
+        machine.fail("lsmod | grep nf_tables")
+  '';
+})

pkgs/os-specific/linux/lxcfs/default.nix

@@ -1,4 +1,5 @@
 { config, stdenv, fetchFromGitHub, autoreconfHook, pkgconfig, help2man, fuse
+, utillinux, makeWrapper
 , enableDebugBuild ? config.lxcfs.enableDebugBuild or false }:
 
 with stdenv.lib;
@@ -13,7 +14,7 @@ stdenv.mkDerivation rec {
   };
 
   nativeBuildInputs = [ pkgconfig help2man autoreconfHook ];
-  buildInputs = [ fuse ];
+  buildInputs = [ fuse makeWrapper ];
 
   preConfigure = stdenv.lib.optionalString enableDebugBuild ''
     sed -i 's,#AM_CFLAGS += -DDEBUG,AM_CFLAGS += -DDEBUG,' Makefile.am
@@ -27,6 +28,12 @@ stdenv.mkDerivation rec {
 
   installFlags = [ "SYSTEMD_UNIT_DIR=\${out}/lib/systemd" ];
 
+  postInstall = ''
+    # `mount` hook requires access to the `mount` command from `utillinux`:
+    wrapProgram "$out/share/lxcfs/lxc.mount.hook" \
+      --prefix PATH : "${utillinux}/bin"
+  '';
+
   postFixup = ''
     # liblxcfs.so is reloaded with dlopen()
     patchelf --set-rpath "$(patchelf --print-rpath "$out/bin/lxcfs"):$out/lib" "$out/bin/lxcfs"

pkgs/tools/admin/lxd/default.nix

@@ -1,13 +1,21 @@
 { stdenv, hwdata, pkgconfig, lxc, buildGoPackage, fetchurl
 , makeWrapper, acl, rsync, gnutar, xz, btrfs-progs, gzip, dnsmasq
-, squashfsTools, iproute, iptables, ebtables, libcap, libco-canonical, dqlite
-, raft-canonical, sqlite-replication, udev
+, squashfsTools, iproute, iptables, ebtables, iptables-nftables-compat, libcap
+, libco-canonical, dqlite, raft-canonical, sqlite-replication, udev
 , writeShellScriptBin, apparmor-profiles, apparmor-parser
 , criu
 , bash
 , installShellFiles
+, nftablesSupport ? false
 }:
 
+let
+  networkPkgs = if nftablesSupport then
+    [ iptables-nftables-compat ]
+  else
+    [ iptables ebtables ];
+
+in
 buildGoPackage rec {
   pname = "lxd";
   version = "4.1";
@@ -38,12 +46,14 @@ buildGoPackage rec {
     # test binaries, code generation
     rm $out/bin/{deps,macaroon-identity,generate}
 
-    wrapProgram $out/bin/lxd --prefix PATH : ${stdenv.lib.makeBinPath [
-      acl rsync gnutar xz btrfs-progs gzip dnsmasq squashfsTools iproute iptables ebtables bash criu
-      (writeShellScriptBin "apparmor_parser" ''
-        exec '${apparmor-parser}/bin/apparmor_parser' -I '${apparmor-profiles}/etc/apparmor.d' "$@"
-      '')
-    ]}
+    wrapProgram $out/bin/lxd --prefix PATH : ${stdenv.lib.makeBinPath (
+      networkPkgs
+      ++ [ acl rsync gnutar xz btrfs-progs gzip dnsmasq squashfsTools iproute bash criu ]
+      ++ [ (writeShellScriptBin "apparmor_parser" ''
+             exec '${apparmor-parser}/bin/apparmor_parser' -I '${apparmor-profiles}/etc/apparmor.d' "$@"
+           '') ]
+      )
+    }
 
     installShellCompletion --bash go/src/github.com/lxc/lxd/scripts/bash/lxd-client
   '';