Use modular NixOS test framework

docs/modules/ROOT/pages/reference/nix-functions/effectVMTest.adoc

@@ -1,26 +1,21 @@
 
 = `effectVMTest`
 
-_effectVMTest {two-colons} AttrSet -> Derivation_
+_effectVMTest {two-colons} Module -> Derivation_
 
 Create offline tests for effect functions.
 
-This returns a derivation that tests an effect using QEMU and `nixosTest`.
+This returns a derivation that tests one or more effects using a NixOS VM test.
 
 Full example: https://github.com/hercules-ci/hercules-ci-effects/blob/master/effects/ssh/test.nix[test for the `ssh` function]
 
-[[parameters]]
-== Parameters
+[[parameter]]
+== Parameter
 
-[[param-name]]
-=== `name`
+The only parameter is a test module. It is mixed in with the https://nixos.org/manual/nixos/unstable/index.html#sec-nixos-tests[NixOS VM test framework] and `hercules-ci-effects`' own options.
 
-Name of the test. Appears in the store path and in the dashboard page with the log.
-
-Default: `"unnamed"`
-
-[[param-effects]]
-=== `effects`
+[[option-effects]]
+=== `effects.<name>`
 
 An attribute set of effects. These can be run with
 
@@ -32,6 +27,11 @@ effectVMTest {
     hello = mkEffect { /* ... */ };
   };
 
+  nodes.foo = {
+    # ... A NixOS configuration running SSH or
+    #     any other relevant service. ...
+  }
+
   testScript = ''
 
     # ... setup ...
@@ -44,17 +44,11 @@ effectVMTest {
 }
 ```
 
-[[param-nodes]]
-=== `nodes`
-
-An attribute set of NixOS configurations. Merged into this is a node named `agent`, which is responsible for running the effects.
-
-See https://nixos.org/manual/nixos/stable/index.html#sec-writing-nixos-tests[NixOS manual: Writing Tests].
 
-[[param-secrets]]
-=== `secrets`
+[[option-secrets]]
+=== `secrets.<name>`
 
-Secrets data that is made available to the effects.
+Secrets to make available to the effects. These are added to the store, so don't copy real-world secrets into this!
 
 Example:
 
@@ -74,7 +68,18 @@ effectVMTest {
 }
 ```
 
-[[param-testScript]]
-=== `testScript`
+[[option-nodes-agent]]
+=== `nodes.agent`
+
+The `hercules-ci-effects` framework adds this VM. It is responsible for running the effects.
+You could modify the settings of this node, but most settings have no effect and the effects,
+because effects run in a sandbox.
+
+Other nodes can be defined in `nodes.<name>` as usual.
+
+See the https://nixos.org/manual/nixos/unstable/index.html#sec-nixos-tests[NixOS VM test framework documentation].
+
+[[options]]
+=== `*`
 
-Function to python statements. See https://nixos.org/manual/nixos/stable/index.html#sec-writing-nixos-tests[NixOS manual: Writing Tests].
+For the other options, refer to the https://nixos.org/manual/nixos/unstable/index.html#sec-nixos-tests[NixOS VM test framework documentation].

effects/effect-vm-test/default.nix

@@ -1,75 +1,14 @@
-{ lib, nixosTest, hci, runtimeShell, writeScriptBin, writeTextFile, writeText, runCommand, writeReferencesToFile }:
+{ lib, pkgs, ... }:
 let
   inherit (lib) isFunction mapAttrsToList;
 
-  # TODO: remove when next hci release is in Nixpkgs
-  hci =
-    let flake = builtins.getFlake "git+https://github.com/hercules-ci/hercules-ci-agent?ref=master&rev=6e298a833dc5321f7f9ff25bc243e4d7c65d928d";
-    in flake.packages.x86_64-linux.hercules-ci-cli;
-
-  # TODO: use Nixpkgs lib.toFunction
-  toFunction =
-    # Any value
-    v:
-    if isFunction v
-    then v
-    else k: v;
-
-  wrapEffect = name: effect:
-    let
-      eff = effect.overrideAttrs (o: {
-        # Work around Nix bug with unsafeDiscardOutputDependency, probably in libstore.
-        makeNixSandboxBuildSucceed = true;
-      });
-    in
-    writeScriptBin "effect-${name}" ''
-      #!${runtimeShell}
-      # retaining deps: ${eff.inputDerivation}
-      hci effect run --no-token --project testforge/testorg/testrepo --as-branch main ${eff.drvPath}
-    '';
-
-  /*
-    Return a store path with a closure containing everything including
-    derivations and all build dependency outputs, all the way down.
-  */
-  allDrvOutputs = pkg:
-    let name = "allDrvOutputs-${pkg.pname or pkg.name or "unknown"}";
-    in
-    runCommand name { refs = writeReferencesToFile pkg.drvPath; } ''
-      touch $out
-      while read ref; do
-        case $ref in
-          *.drv)
-            cat $ref >>$out
-            ;;
-        esac
-      done <$refs
-    '';
+  nixos-lib = import (pkgs.path + "/nixos/lib") { inherit lib; };
 
 in
 
-{ name ? "unnamed", effects, nodes, secrets ? { }, testScript }:
-let
-  secrets2 = lib.mapAttrs
-    (k: v:
-      lib.throwIfNot (v?data) "secret `${k}` does not have a `data` attribute in test `${name}`" (
-        { kind = "Secret"; condition = { and = [ ]; }; } // v
-      )
-    )
-    secrets;
-  secretsFile = writeText "fake-secrets-${name}" (builtins.toJSON secrets2);
-in
-nixosTest {
-  name = "effect-${name}";
-  nodes = nodes // {
-    agent = {
-      imports = [ nodes.agent or { } ];
-      environment.systemPackages = [ hci ] ++ mapAttrsToList wrapEffect effects;
-      # Might actually want to use `hci secret add` instead?
-      # That will support dynamic secrets, like a host key that's
-      # generated on the host.
-      environment.variables.HERCULES_CI_SECRETS_JSON = "${secretsFile}";
-    };
+module: nixos-lib.runTest {
+  imports = [ ./effects-module.nix module ];
+  config = {
+    hostPkgs = pkgs;
   };
-  inherit testScript;
 }

effects/effect-vm-test/effects-module.nix

@@ -0,0 +1,91 @@
+test@
+{ config
+, lib
+, pkgs
+, ...
+}:
+# pkgs, hci, runtimeShell, writeScriptBin, writeTextFile, writeText, runCommand, writeReferencesToFile }:
+let
+  inherit (lib) isFunction mapAttrsToList mkOption types;
+
+  nixos-lib = import (pkgs.path + "/nixos/lib") { inherit lib; };
+
+  # TODO: remove when next hci release is in Nixpkgs
+  hci =
+    let flake = builtins.getFlake "git+https://github.com/hercules-ci/hercules-ci-agent?ref=master&rev=6e298a833dc5321f7f9ff25bc243e4d7c65d928d";
+    in flake.packages.x86_64-linux.hercules-ci-cli;
+
+  wrapEffect = name: effect:
+    let
+      eff = effect.overrideAttrs (o: {
+        # Work around Nix bug with unsafeDiscardOutputDependency, probably in libstore.
+        makeNixSandboxBuildSucceed = true;
+      });
+    in
+    pkgs.writeScriptBin "effect-${name}" ''
+      #!${pkgs.runtimeShell}
+      # retaining deps: ${eff.inputDerivation}
+      hci effect run --no-token --project testforge/testorg/testrepo --as-branch main ${eff.drvPath}
+    '';
+
+  /*
+    Return a store path with a closure containing everything including
+    derivations and all build dependency outputs, all the way down.
+  */
+  allDrvOutputs = pkg:
+    let name = "allDrvOutputs-${pkg.pname or pkg.name or "unknown"}";
+    in
+    pkgs.runCommand name { refs = pkgs.writeReferencesToFile pkg.drvPath; } ''
+      touch $out
+      while read ref; do
+        case $ref in
+          *.drv)
+            cat $ref >>$out
+            ;;
+        esac
+      done <$refs
+    '';
+
+  secrets2 = lib.mapAttrs
+    (k: v:
+      lib.throwIfNot (v?data) "secret `${k}` does not have a `data` attribute in test `${config.name}`" (
+        { kind = "Secret"; condition = { and = [ ]; }; } // v
+      )
+    )
+    config.secrets;
+  secretsFile = pkgs.writeText "fake-secrets-${config.name}" (builtins.toJSON secrets2);
+
+in
+{
+  options = {
+    effects = mkOption {
+      description = ''
+        An attribute set of effects.
+
+        The attribute name (referred to as `<name>`) translates to a command that is runnable from the {option}`testScript` as
+
+        ```python
+            agent.succeed("effect-<name>")
+        ```
+      '';
+      type = types.lazyAttrsOf types.package;
+    };
+
+    secrets = mkOption {
+      description = ''
+        A collection of secrets available on the mock agent.
+      '';
+      type = types.lazyAttrsOf (types.lazyAttrsOf types.raw);
+    };
+  };
+
+  config = {
+    nodes.agent = {
+      environment.systemPackages = [ hci ] ++ mapAttrsToList wrapEffect test.config.effects;
+      # Might actually want to use `hci secret add` instead?
+      # That will support dynamic secrets, like a host key that's
+      # generated on the host.
+      environment.variables.HERCULES_CI_SECRETS_JSON = "${secretsFile}";
+    };
+  };
+}

effects/ssh/test.nix

@@ -5,33 +5,9 @@ let
 
 in
 effectVMTest {
+  imports = [ ../testsupport/dns.nix ];
   name = "ssh";
   nodes = {
-    ns = { nodes, ... }: {
-      networking.firewall.allowedUDPPorts = [ 53 ];
-      services.bind.enable = true;
-      services.bind.extraOptions = "empty-zones-enable no;";
-      services.bind.zones = [{
-        name = ".";
-        master = true;
-        file = writeText "root.zone" ''
-          $TTL 3600
-          . IN SOA ns. ns. ( 1 8 2 4 1 )
-          . IN NS ns.
-          ${concatMapStringsSep
-            "\n"
-            (node: "${node.config.networking.hostName}. IN A ${node.config.networking.primaryIPAddress}")
-            (builtins.attrValues nodes)
-          }
-        '';
-      }];
-    };
-    agent = { nodes, ... }: {
-      networking.dhcpcd.enable = false;
-      environment.etc."resolv.conf".text = ''
-        nameserver ${nodes.ns.config.networking.primaryIPAddress}
-      '';
-    };
     target = { ... }: {
       environment.etc."unsafe-ssh/host" = {
         source = ./test/host;
@@ -74,16 +50,12 @@ effectVMTest {
   };
   testScript = { nodes, ... }: ''
     start_all()
-    ns.wait_for_unit("bind.service")
-    ns.wait_for_open_port(53)
+    dns.wait_for_unit("bind.service")
+    dns.wait_for_open_port(53)
     agent.wait_for_unit("multi-user.target")
     target.wait_for_unit("sshd.service")
     target.wait_for_open_port(22)
 
-    agent.succeed("cat /etc/hosts >/dev/console")
-    agent.succeed("cat /etc/resolv.conf >/dev/console")
-    agent.succeed("host target ${nodes.ns.config.networking.primaryIPAddress}")
-    agent.succeed("host target")
     agent.succeed("effect-ssh1")
     target.succeed("""[[ "$(cat ~/it-worked)" == it\ worked ]]""")
     target.succeed("grep Hello <~/greeting")

effects/testsupport/dns.nix

@@ -0,0 +1,58 @@
+/*
+  A NixOS test module that provides a DNS server and configures it on all nodes.
+*/
+{ config, lib, ... }:
+let
+
+  inherit (lib)
+    concatMapStringsSep
+    mkOption types
+    ;
+
+  cfg = config.dns;
+
+in
+{
+
+  options = {
+    dns.nodeName = mkOption {
+      description = ''
+        The `<name>` in `nodes.<name>` for the DNS server.
+      '';
+      type = types.str;
+      default = "dns";
+    };
+  };
+
+  config = {
+
+    nodes.${cfg.nodeName} = { nodes, pkgs, ... }: {
+      networking.firewall.allowedUDPPorts = [ 53 ];
+      services.bind.enable = true;
+      services.bind.extraOptions = "empty-zones-enable no;";
+      services.bind.zones = [{
+        name = ".";
+        master = true;
+        file = pkgs.writeText "root.zone" ''
+          $TTL 3600
+          . IN SOA ${cfg.nodeName}. ${cfg.nodeName}. ( 1 8 2 4 1 )
+          . IN NS ${cfg.nodeName}.
+          ${concatMapStringsSep
+            "\n"
+            (node: "${node.networking.hostName}. IN A ${node.networking.primaryIPAddress}")
+            (builtins.attrValues nodes)
+          }
+        '';
+      }];
+    };
+
+    defaults = { nodes, ... }: {
+      networking.dhcpcd.enable = false;
+      environment.etc."resolv.conf".text = ''
+        nameserver ${nodes.${cfg.nodeName}.networking.primaryIPAddress}
+      '';
+    };
+
+  };
+
+}