From 427f852621b3c7da7251708a87b3fcbf1a6f840a Mon Sep 17 00:00:00 2001 From: Jack Cummings Date: Mon, 2 Sep 2013 00:47:15 -0700 Subject: [PATCH 1/8] document how to install NixOS to a zfs root --- doc/manual/installation.xml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/doc/manual/installation.xml b/doc/manual/installation.xml index 3068fa5cb9..73b909db93 100644 --- a/doc/manual/installation.xml +++ b/doc/manual/installation.xml @@ -84,12 +84,33 @@ $ lvcreate --size 1G --name smalldisk MyVolGroup For creating software RAID devices, use mdadm. + For creating ZFS pools, the zfs commands, e.g., + + +$ zpool create zroot /dev/sda2 .... +$ zfs create zroot/root + + + Mount the target file system on which NixOS should - be installed on /mnt. + be installed on /mnt. + + + If you're using a ZFS root, then you'll need to mark it as a legacy mount for the NixOS initrd to properly mount it. + ZFS root also requires requires a separate /boot partition. + +$ zfs set mountpoint=legacy zroot/root +$ mount -t zfs zroot/root /mnt +$ mkfs.ext3 -L boot /dev/sda1 +$ mkdir /mnt/boot +$ mount LABEL=boot /mnt/boot + + + From 10ca49f9553a44795ae1e25ac4d85c7229a569b6 Mon Sep 17 00:00:00 2001 From: Jack Cummings Date: Mon, 2 Sep 2013 00:47:40 -0700 Subject: [PATCH 2/8] installer tests for ZFS root (zroot) --- release.nix | 1 + tests/default.nix | 1 + tests/installer.nix | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/release.nix b/release.nix index a6b80be0ea..7115daf4c3 100644 --- a/release.nix +++ b/release.nix @@ -249,6 +249,7 @@ in { installer.rebuildCD = runTest (t: t.installer.rebuildCD.test); installer.separateBoot = runTest (t: t.installer.separateBoot.test); installer.simple = runTest (t: t.installer.simple.test); + installer.zroot = runTest (t: t.installer.zroot.test); #installer.swraid = runTest (t: t.installer.swraid.test); ipv6 = runTest (t: t.ipv6.test); kde4 = runTest (t: t.kde4.test); diff --git a/tests/default.nix b/tests/default.nix index 0822c93e81..1a1b937a2b 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -29,4 +29,5 @@ with import ../lib/testing.nix { inherit system minimal; }; tomcat = makeTest (import ./tomcat.nix); trac = makeTest (import ./trac.nix); xfce = makeTest (import ./xfce.nix); + ztest = makeTest (import ./ztest.nix); } diff --git a/tests/installer.nix b/tests/installer.nix index 8488726efe..29d72b927b 100644 --- a/tests/installer.nix +++ b/tests/installer.nix @@ -57,6 +57,12 @@ let } ''; + zRootFS = + '' + fileSystems."/".device = "zroot/root"; + fileSystems."/".fsType = "zfs"; + ''; + rootFS = '' fileSystems."/".device = "/dev/disk/by-label/nixos"; @@ -301,6 +307,32 @@ in { fileSystems = rootFS + bootFS; }; + # separate boot/zfs root partition + zroot = makeTest + { createPartitions = + '' + $machine->succeed( + "parted /dev/vda mklabel msdos", + "parted /dev/vda -- mkpart primary 1M 50MB", # /boot + "parted /dev/vda -- mkpart primary 50MB 2048M", + "udevadm settle", + "zpool create zroot /dev/vda2", # start a pool + "zfs create zroot/root", # create root + "zfs set mountpoint=legacy zroot/root", # mount this fs with 'mount' instead of 'zfs mount' + "zfs set compression=lz4 zroot/root", # compress things a bit, so we don't run out of space. + "mount -t zfs zroot/root /mnt", + "zfs create -V 512M -b 4K zroot/swap", # create a swap zvol + "mkswap -L swap /dev/zvol/zroot/swap", + "swapon -L swap", + "mkfs.ext3 -L boot /dev/vda1", # separate boot partition + "mkdir /mnt/boot", + "mount LABEL=boot /mnt/boot", + "udevadm settle", + ); + ''; + fileSystems = zRootFS + bootFS; + }; + # Test a basic install using GRUB 1. grub1 = makeTest { createPartitions = From e696210a05579401fe9973dca35d2f80d2ffaa62 Mon Sep 17 00:00:00 2001 From: Jack Cummings Date: Mon, 2 Sep 2013 00:54:03 -0700 Subject: [PATCH 3/8] zfs task cleanup Put the zfs, ztest, zdb and zpool binaries in ../bin so that they get patchelf'd by the initrd builder Move restartIfChanged attribute to where it is being heeded. Ignore failures in zfs mount, and zpool imports. Make zpool import wanted by the filesystem target, otherwise it might not happen. --- modules/tasks/filesystems/zfs.nix | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/modules/tasks/filesystems/zfs.nix b/modules/tasks/filesystems/zfs.nix index 590eb00202..8d5de9a11a 100644 --- a/modules/tasks/filesystems/zfs.nix +++ b/modules/tasks/filesystems/zfs.nix @@ -50,27 +50,31 @@ in boot.initrd = mkIf inInitrd { kernelModules = [ "spl" "zfs" ] ; + # zfs uses libs from: zfs, glibc, utillinux, zlib, gcc and glibc. + # stage-1.nix provides: glibc utlilinux gcc and glibc extraUtilsCommands = '' - cp -v ${kernel.zfs}/sbin/zfs $out/sbin - cp -v ${kernel.zfs}/sbin/zdb $out/sbin - cp -v ${kernel.zfs}/sbin/zpool $out/sbin + cp -v ${kernel.zfs}/sbin/zfs $out/bin + cp -v ${kernel.zfs}/sbin/zdb $out/bin + cp -v ${kernel.zfs}/sbin/zpool $out/bin + cp -pdv ${kernel.zfs}/lib/lib*.so.* $out/lib + cp -pdv ${pkgs.zlib}/lib/lib*.so.* $out/lib ''; postDeviceCommands = '' zpool import -f -a -d /dev - zfs mount -a ''; }; - systemd.services."zpool-import" = { + systemd.services."zpool-import" = mkIf inSystem { description = "Import zpools"; after = [ "systemd-udev-settle.service" ]; + wantedBy = [ "local-fs.target" ]; + restartIfChanged = false; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; - restartIfChanged = false; - ExecStart = "${kernel.zfs}/sbin/zpool import -f -a -d /dev"; + ExecStart = "-${kernel.zfs}/sbin/zpool import -d /dev -f -a"; # XXX: allow failure? }; }; @@ -78,12 +82,12 @@ in description = "Mount zfs volumes"; after = [ "zpool-import.service" ]; wantedBy = [ "local-fs.target" ]; + restartIfChanged = false; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; - restartIfChanged = false; - ExecStart = "${kernel.zfs}/sbin/zfs mount -a"; - ExecStop = "${kernel.zfs}/sbin/zfs umount -a"; + ExecStart = "-${kernel.zfs}/sbin/zfs mount -a"; # XXX: allow failure? + ExecStop = "-${kernel.zfs}/sbin/zfs umount -a"; # XXX: allow failure? }; }; From 146106763b67013823ce109abb1b2e73cff20d7b Mon Sep 17 00:00:00 2001 From: Jack Cummings Date: Fri, 6 Sep 2013 21:59:46 -0700 Subject: [PATCH 4/8] zfs does not work with less than 768M of memory --- lib/test-driver/Machine.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/test-driver/Machine.pm b/lib/test-driver/Machine.pm index c56e650ed3..3e1fec0eb6 100644 --- a/lib/test-driver/Machine.pm +++ b/lib/test-driver/Machine.pm @@ -30,7 +30,7 @@ sub new { if (!$startCommand) { # !!! merge with qemu-vm.nix. $startCommand = - "qemu-kvm -m 384 " . + "qemu-kvm -m 768 " . "-net nic,model=virtio \$QEMU_OPTS "; my $iface = $args->{hdaInterface} || "virtio"; $startCommand .= "-drive file=" . Cwd::abs_path($args->{hda}) . ",if=$iface,boot=on,werror=report " From 0425ab7e6a604574e0793959a0c9406ccd1e11f8 Mon Sep 17 00:00:00 2001 From: Jack Cummings Date: Fri, 6 Sep 2013 22:00:03 -0700 Subject: [PATCH 5/8] support zfs in the iso --- modules/profiles/base.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/profiles/base.nix b/modules/profiles/base.nix index 20b808c29e..63bc59d374 100644 --- a/modules/profiles/base.nix +++ b/modules/profiles/base.nix @@ -50,6 +50,6 @@ ]; # Include support for various filesystems. - boot.supportedFilesystems = [ "btrfs" "reiserfs" "vfat" ]; + boot.supportedFilesystems = [ "btrfs" "reiserfs" "vfat" "zfs" ]; } From 5fdb8b7464b2d8cdb1c856aa14b14c1f88d74408 Mon Sep 17 00:00:00 2001 From: Jack Cummings Date: Fri, 6 Sep 2013 22:00:36 -0700 Subject: [PATCH 6/8] ztest seems to expose some bugs in writeback mode --- modules/virtualisation/qemu-vm.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/virtualisation/qemu-vm.nix b/modules/virtualisation/qemu-vm.nix index 61d5d20f3b..24702a54fc 100644 --- a/modules/virtualisation/qemu-vm.nix +++ b/modules/virtualisation/qemu-vm.nix @@ -199,11 +199,11 @@ let -virtfs local,path=$TMPDIR/xchg,security_model=none,mount_tag=xchg \ -virtfs local,path=''${SHARED_DIR:-$TMPDIR/xchg},security_model=none,mount_tag=shared \ ${if cfg.useBootLoader then '' - -drive index=0,id=drive1,file=$NIX_DISK_IMAGE,if=virtio,cache=writeback,werror=report \ + -drive index=0,id=drive1,file=$NIX_DISK_IMAGE,if=virtio,cache=writethrough,werror=report \ -drive index=1,id=drive2,file=${bootDisk}/disk.img,if=virtio,readonly \ -boot menu=on '' else '' - -drive file=$NIX_DISK_IMAGE,if=virtio,cache=writeback,werror=report \ + -drive file=$NIX_DISK_IMAGE,if=virtio,cache=writethrough,werror=report \ -kernel ${config.system.build.toplevel}/kernel \ -initrd ${config.system.build.toplevel}/initrd \ -append "$(cat ${config.system.build.toplevel}/kernel-params) init=${config.system.build.toplevel}/init regInfo=${regInfo} ${kernelConsole} $QEMU_KERNEL_PARAMS" \ From 76616c694aea85062dc561aae987fe59365fcf79 Mon Sep 17 00:00:00 2001 From: Jack Cummings Date: Fri, 6 Sep 2013 22:01:14 -0700 Subject: [PATCH 7/8] tweak swap zvol for lower memory Remove caching and compression on the zvol used for swap, in the event of a low memory condition, this means less chance of an expensive RMW . --- tests/installer.nix | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/installer.nix b/tests/installer.nix index 29d72b927b..37431fe6c1 100644 --- a/tests/installer.nix +++ b/tests/installer.nix @@ -59,6 +59,7 @@ let zRootFS = '' + boot.initrd.supportedFilesystems = [ "ext3" "zfs" ]; fileSystems."/".device = "zroot/root"; fileSystems."/".fsType = "zfs"; ''; @@ -316,12 +317,15 @@ in { "parted /dev/vda -- mkpart primary 1M 50MB", # /boot "parted /dev/vda -- mkpart primary 50MB 2048M", "udevadm settle", - "zpool create zroot /dev/vda2", # start a pool - "zfs create zroot/root", # create root - "zfs set mountpoint=legacy zroot/root", # mount this fs with 'mount' instead of 'zfs mount' - "zfs set compression=lz4 zroot/root", # compress things a bit, so we don't run out of space. - "mount -t zfs zroot/root /mnt", - "zfs create -V 512M -b 4K zroot/swap", # create a swap zvol + "zpool create zroot /dev/vda2", # start a pool + "zfs create zroot/root", # create root + "zfs set mountpoint=legacy zroot/root", # mount this fs with 'mount' instead of 'zfs mount' + "zfs set compression=lz4 zroot/root", # compress things a bit, so we don't run out of space. + "mount -t zfs zroot/root /mnt", + "zfs create -V 1G -b 4K zroot/swap", # create a swap zvol + "zfs set compression=off zroot/swap", # don't compress swap + "zfs set primarycache=metadata zroot/swap", # only cache metadata + "zfs set sync=always zroot/swap", "mkswap -L swap /dev/zvol/zroot/swap", "swapon -L swap", "mkfs.ext3 -L boot /dev/vda1", # separate boot partition From 335bef03d819f9f448c97d8536bfc3135bdea211 Mon Sep 17 00:00:00 2001 From: Jack Cummings Date: Fri, 6 Sep 2013 22:08:32 -0700 Subject: [PATCH 8/8] zfs unit tests --- tests/ztest.nix | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/ztest.nix diff --git a/tests/ztest.nix b/tests/ztest.nix new file mode 100644 index 0000000000..8f2db0fc9c --- /dev/null +++ b/tests/ztest.nix @@ -0,0 +1,33 @@ +{ pkgs, ... }: + +{ + nodes = { + machine = + { pkgs, config, ... }: + + { + virtualisation.emptyDiskImages = [ 4096 ]; + boot.supportedFilesystems = [ "zfs" "ext4" ] ; + }; + }; + + testScript = '' + startAll; + + $machine->waitForUnit("default.target"); + + # this is a work in progress, it currently fails with: + # pack/bigT mismatch in 0x7f5ff010cf60/0x7f5ff068fea8 + # probably some disk io synchronization problems. + + # we don't directly create a zfs filesystem here we create a ext3 + # filesystem and the the zfs unit tests use it.. + + $machine->succeed("mkfs.ext3 -L ztest /dev/vdb"); + $machine->succeed("mkdir /ztest"); + $machine->succeed("mount LABEL=ztest /ztest"); + $machine->succeed("echo Running ZFS unit tests, this may take a while..."); + $machine->succeed("ztest -f /ztest -t1 -VVV"); # single threaded + $machine->shutdown; + ''; +}