Reworking test infrastructure

lib/default.nix

@@ -32,6 +32,7 @@ let
   # domain-specific
   sandbox = import ./sandbox.nix;
   fetchers = import ./fetchers.nix;
+  testing = import ./testing {};
 
 in
   { inherit trivial
@@ -40,7 +41,7 @@ in
             modules options types
             licenses platforms systems
             debug misc
-            sandbox fetchers;
+            sandbox fetchers testing;
   }
   # !!! don't include everything at top-level; perhaps only the most
   # commonly used functions.

lib/testing/default.nix

@@ -0,0 +1,75 @@
+{ supportedSystems ? [ builtins.currentSystem ] }:
+
+with {
+  inherit (builtins) getAttr;
+  inherit (import ../attrsets.nix) mapAttrs genAttrs isDerivation;
+  inherit (import ../customisation.nix) hydraJob;
+  inherit (import ../trivial.nix) id;
+};
+
+rec {
+
+  /* Call a test file
+
+     The called test file should contain a function that returns:
+     - a single test set
+     - a set of tests sets
+     - a test derivation, eg runInMachine
+
+     Examples:
+
+     a. Call a test:
+       callTest ./nixos/tests/simple.nix {};
+
+     b. Call a test for only x86_64-linux:
+       callTest ./nixos/tests/simple.nix { systems = [ "x86_64-linux" ]; };
+
+     c. Call a test with custom parameters:
+       callTest ./nixos/tests/nfs.nix { version = 3; };
+
+     d. Call a test from the command line with nix-build:
+       $ nix-build -E 'with import <nixpkgs/lib/testing> {}; callTest ./nixos/tests/simple.nix {}'
+
+  */
+
+  callTest = file: { systems ? supportedSystems, interactive ? false, ... } @ args:
+    let
+      # raw test function
+      testFn = import file;
+      # pkgs needed to dummy evaluate the test, not used in the final test
+      pkgs = import ../.. { system = "x86_64-linux"; };
+      # basic arguments
+      testArgs = rec { inherit pkgs; inherit (pkgs) lib; } // args;
+      # test applied with basic arguments
+      dummyTest = testFn testArgs;
+      # check if the test is a derivation like in runInMachine
+      drvTest = isDerivation dummyTest;
+      # to check if it is a single test or multiple tests
+      singleTest = dummyTest ? "testScript";
+      # generate the tests with makeTest function
+      mkTest = { system ? builtins.currentSystem, ... } @ args:
+        let testLib = import ./testing.nix { inherit system; };
+            # pkgs coming from testing.nix for the system tested
+            pkgsSet = { inherit testLib pkgs; };
+            test = testFn (args // pkgsSet);
+        in if drvTest then test
+              else if singleTest
+                   then testLib.makeTest test
+                   else mapAttrs (k: v: testLib.makeTest v) test;
+      # function to prepare the test
+      prepTestFn = if interactive
+                   then (getAttr "driver")
+                   else hydraJob;
+      # check unsupported cases
+      runCheck = f: if (drvTest && interactive)
+                 then builtins.abort "Derivation tests cannot be run interactively"
+                 else f;
+      # generate test jobs for systems
+      testForSys = f: genAttrs systems (system: prepTestFn( f( mkTest({ inherit system; } // testArgs)) ) );
+      # fully prepared test
+      test = if (drvTest || singleTest)
+                then testForSys id
+                else mapAttrs (k: _: testForSys (getAttr k)) dummyTest;
+      in runCheck test;
+
+}

nixos/lib/testing.nix → lib/testing/testing.nix

@@ -1,6 +1,6 @@
 { system, minimal ? false, config ? {} }:
 
-with import ./build-vms.nix { inherit system minimal config; };
+with import ../../nixos/lib/build-vms.nix { inherit system minimal config; };
 with pkgs;
 
 rec {

nixos/doc/manual/development/meta-attributes.xml

@@ -14,8 +14,8 @@
 
 <para><literal>meta</literal> is a top level attribute like
   <literal>options</literal> and <literal>config</literal>. Available
-  meta-attributes are <literal>maintainers</literal> and
-  <literal>doc</literal>.</para>
+  meta-attributes are <literal>maintainers</literal>,
+  <literal>doc</literal> and <literal>tests</literal>.</para>
 
 <para>Each of the meta-attributes must be defined at most once per module
   file.</para>
@@ -35,6 +35,10 @@
     maintainers = with lib.maintainers; [ ericsagnes ]; <co
       xml:id='modules-meta-1' />
     doc = ./default.xml; <co xml:id='modules-meta-2' />
+    tests = { <co xml:id='modules-meta-3' />
+      containers-ipv4 = ./tests/containers-ipv4.nix;
+      containers-ipv6 = ./tests/containers-ipv6.nix;
+    };
   };
 }
 </programlisting>
@@ -57,6 +61,19 @@
   <programlisting>$ nix-build nixos/release.nix -A manual</programlisting>
  </callout>
 
+ <callout arearefs='modules-meta-3'>
+  <para>
+    <varname>tests</varname> is a set of which keys are test names and values
+    point to a test file. All the tests declared via this attribute are
+    automatically included in
+    <filename xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/release.nix">release.nix</filename>'s
+    <literal>tests.module</literal> set.
+    It is important to check that newly added and modified test are not
+    breaking by running them:
+  </para>
+  <programlisting>$ nix-build nixos/release.nix -A tests.module.TEST_NAME</programlisting>
+ </callout>
+
 </calloutlist>
 
 </section>

nixos/doc/manual/development/running-nixos-tests-interactively.xml

@@ -9,7 +9,7 @@
 particularly useful when developing or debugging a test:
 
 <screen>
-$ nix-build nixos/tests/login.nix -A driver
+$ nix-run-test --interactive nixos/tests/login.nix
 $ ./result/bin/nixos-test-driver
 starting VDE switch for network 1
 &gt;
@@ -31,7 +31,7 @@ test (e.g. to debug the test script).</para>
 <para>To just start and experiment with the VMs, run:
 
 <screen>
-$ nix-build nixos/tests/login.nix -A driver
+$ nix-run-test --interactive nixos/tests/login.nix
 $ ./result/bin/nixos-run-vms
 </screen>
 

nixos/doc/manual/development/running-nixos-tests.xml

@@ -6,20 +6,21 @@
 
 <title>Running Tests</title>
 
-<para>You can run tests using <command>nix-build</command>. For
-example, to run the test <filename
+<para>You can run tests using <command>nix-run-test</command>.
+<command>nix-run-test</command> is available in nixpkgs.
+For example, to run the test <filename
 xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/login.nix">login.nix</filename>,
 you just do:
 
 <screen>
-$ nix-build '&lt;nixpkgs/nixos/tests/login.nix>'
+$ nix-run-test '&lt;nixpkgs/nixos/tests/login.nix>'
 </screen>
 
 or, if you don’t want to rely on <envar>NIX_PATH</envar>:
 
 <screen>
 $ cd /my/nixpkgs/nixos/tests
-$ nix-build login.nix
+$ nix-run-test login.nix
 …
 running the VM test script
 machine: QEMU running (pid 8841)

nixos/doc/manual/development/writing-nixos-tests.xml

@@ -9,7 +9,8 @@
 <para>A NixOS test is a Nix expression that has the following structure:
 
 <programlisting>
-import ./make-test.nix {
+{ pkgs, ... }:
+{
 
   # Either the configuration of a single machine:
   machine =
@@ -266,4 +267,4 @@ startAll;
 
 </para>
 
-</section>
+</section>

nixos/modules/installer/tools/nixos-build-vms/build-vms.nix

@@ -4,6 +4,6 @@
 
 let nodes = import networkExpr; in
 
-with import ../../../../lib/testing.nix { inherit system; };
+with import ../../../../../lib/testing/testing.nix { inherit system; };
 
 (makeTest { inherit nodes; testScript = ""; }).driver

nixos/modules/misc/meta.nix

@@ -22,9 +22,9 @@ let
             [{ inherit (def) file; value = def'; }]) def.value) defs));
   };
 
-  docFile = types.path // {
+  file = types.path // {
     # Returns tuples of
-    #   { file = "module location"; value = <path/to/doc.xml>; }
+    #   { file = "module location"; value = <path/to/file.ext>; }
     merge = loc: defs: defs;
   };
 in
@@ -45,7 +45,7 @@ in
       };
 
       doc = mkOption {
-        type = docFile;
+        type = file;
         internal = true;
         example = "./meta.xml";
         description = ''
@@ -54,6 +54,16 @@ in
         '';
       };
 
+      tests = mkOption {
+        type = with types; attrsOf file;
+        internal = true;
+        example = literalExample "./test.nix";
+        description = ''
+          Tests for the module. Tests get added to the tests attribute of release.nix with the test name prepended by "module-".
+          This option should be defined at most once per module.
+        '';
+      };
+
     };
   };
 

nixos/modules/security/grsecurity.nix

@@ -23,6 +23,9 @@ in
   meta = {
     maintainers = with maintainers; [ joachifm ];
     doc = ./grsecurity.xml;
+    tests = {
+      grsecurity = ./tests/grsecurity.nix;
+    };
   };
 
   options.security.grsecurity = {

nixos/tests/grsecurity.nix → nixos/modules/security/tests/grsecurity.nix

@@ -1,6 +1,6 @@
 # Basic test to make sure grsecurity works
 
-import ./make-test.nix ({ pkgs, ...} : {
+{ pkgs, ...} : {
   name = "grsecurity";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ copumpkin joachifm ];
@@ -43,4 +43,4 @@ import ./make-test.nix ({ pkgs, ...} : {
       $machine->succeed("[ -c /dev/grsec ]");
     };
   '';
-})
+}

nixos/modules/services/amqp/rabbitmq.nix

@@ -135,4 +135,8 @@ in {
 
   };
 
+  meta.tests = {
+    rabbitmq = ./tests/rabbitmq.nix;
+  };
+
 }

nixos/tests/rabbitmq.nix → nixos/modules/services/amqp/tests/rabbitmq.nix

@@ -1,6 +1,6 @@
 # This test runs rabbitmq and checks if rabbitmq is up and running.
 
-import ./make-test.nix ({ pkgs, ... }: {
+{ pkgs, ... }: {
   name = "rabbitmq";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ eelco chaoflow offline ];
@@ -18,4 +18,4 @@ import ./make-test.nix ({ pkgs, ... }: {
     $one->waitForUnit("rabbitmq.service");
     $one->waitUntilSucceeds("su -s ${pkgs.stdenv.shell} rabbitmq -c \"rabbitmqctl status\"");
   '';
-})
+}

nixos/modules/services/cluster/fleet.nix

@@ -147,4 +147,8 @@ in {
     environment.systemPackages = [ pkgs.fleet ];
     users.extraGroups.fleet.gid = config.ids.gids.fleet;
   };
+
+  meta.tests = {
+    fleet = ./tests/fleet.nix;
+  };
 }

nixos/modules/services/cluster/kubernetes.nix

@@ -632,4 +632,8 @@ in {
     })
 
   ];
+
+  meta.tests = {
+    kubernetes = ./tests/kubernetes.nix;
+  };
 }

nixos/modules/services/cluster/panamax.nix

@@ -153,4 +153,8 @@ in {
     environment.systemPackages = [ panamax_api panamax_ui ];
     users.extraGroups.panamax.gid = config.ids.gids.panamax;
   };
+
+  meta.tests = {
+    panamax = ./tests/panamax.nix;
+  };
 }

nixos/tests/fleet.nix → nixos/modules/services/cluster/tests/fleet.nix

@@ -1,4 +1,4 @@
-import ./make-test.nix ({ pkgs, ...} : rec {
+{ pkgs, ...} : rec {
   name = "simple";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ offline ];
@@ -73,4 +73,4 @@ import ./make-test.nix ({ pkgs, ...} : rec {
       $node1->succeed("fleetctl stop hello.service");
       $node1->succeed("fleetctl destroy hello.service");
     '';
-})
+}

nixos/tests/kubernetes.nix → nixos/modules/services/cluster/tests/kubernetes.nix

@@ -1,6 +1,6 @@
 # This test runs two node kubernetes cluster and checks if simple redis pod works
 
-import ./make-test.nix ({ pkgs, ...} : rec {
+{ pkgs, ...} : rec {
   name = "kubernetes";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ offline ];
@@ -179,4 +179,4 @@ import ./make-test.nix ({ pkgs, ...} : rec {
     }
 
   '';
-})
+}

nixos/tests/panamax.nix → nixos/modules/services/cluster/tests/panamax.nix

@@ -1,4 +1,4 @@
-import ./make-test.nix ({ pkgs, ...} : {
+{ pkgs, ...} : {
   name = "panamax";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ offline ];
@@ -18,4 +18,4 @@ import ./make-test.nix ({ pkgs, ...} : {
       $machine->succeed("curl --fail http://localhost:8888/ > /dev/null");
       $machine->shutdown;
     '';
-})
+}

nixos/modules/services/computing/slurm/slurm.nix

@@ -168,4 +168,8 @@ in
 
   };
 
+  meta.tests = {
+    slurm = ./test.nix;
+  };
+
 }

nixos/tests/slurm.nix → nixos/modules/services/computing/slurm/test.nix

@@ -1,4 +1,4 @@
-import ./make-test.nix ({ pkgs, ... }:
+{ pkgs, ... }:
 let mungekey = "mungeverryweakkeybuteasytointegratoinatest";
     slurmconfig = {
       client.enable = true;
@@ -77,4 +77,4 @@ in {
     $control->succeed("srun -N 3 hostname | sort | uniq | wc -l | xargs test 3 -eq");
   };
   '';
-})
+}

nixos/modules/services/continuous-integration/gocd-agent/default.nix

@@ -202,4 +202,8 @@ in {
       };
     };
   };
+
+  meta.tests = {
+    gocd-agent = ./test.nix;
+  };
 }