refactor: enable parallel ci bats tests by enabling dfx start and bootstrap to pick random webserver ports

.github/CONTRIBUTING.adoc

@@ -30,7 +30,7 @@ dfx 0.5.7-1-gad81116
 
 [source,bash]
 ----
-sdk $ nix-shell -A e2e-tests default.nix
+sdk $ nix-shell -A ${name_of_any_e2e_test} e2e/bats/default.nix
 [nix-shell:~/d/sdk]$ cd e2e/bats
 [nix-shell:~/d/sdk]$ bats *.bash
 ----
@@ -42,9 +42,9 @@ https://github.com/dfinity-lab/ic-ref[reference implementation of the Internet C
 
 [source,bash]
 ----
-sdk $ nix-shell -A e2e-tests-ic-ref default.nix
+sdk $ nix-shell -A ${name_of_any_e2e_test} --arg use_ic_ref true e2e/bats/default.nix
 [nix-shell:~/d/sdk]$ cd e2e/bats
-[nix-shell:~/d/sdk]$ USE_IC_REF=1 bats *.bash
+[nix-shell:~/d/sdk]$ bats *.bash
 ----
 
 ==== Running `dfx` in a Debugger

e2e/bats/bootstrap.bash

@@ -18,10 +18,10 @@ teardown() {
     dfx build
     dfx canister install hello
     ID=$(dfx canister id hello)
-
-    assert_command curl http://localhost:8000/_/candid?canisterId="$ID" -o ./web.txt
+    PORT=$(cat .dfx/webserver-port)
+    assert_command curl http://localhost:"$PORT"/_/candid?canisterId="$ID" -o ./web.txt
     assert_command diff .dfx/local/canisters/hello/hello.did ./web.txt
-    assert_command curl http://localhost:8000/_/candid?canisterId="$ID"\&format=js -o ./web.txt
+    assert_command curl http://localhost:"$PORT"/_/candid?canisterId="$ID"\&format=js -o ./web.txt
     # Relax diff as it's produced by two different compilers.
     assert_command diff --ignore-all-space --ignore-blank-lines .dfx/local/canisters/hello/hello.did.js ./web.txt
 }

e2e/bats/build.bash

@@ -74,8 +74,8 @@ teardown() {
 }
 
 @test "can build a custom canister type" {
-  dfx_start
   install_asset custom_canister
+  dfx_start
   dfx canister create --all
   assert_command dfx build
   assert_match "CUSTOM_CANISTER_BUILD_DONE"
@@ -93,7 +93,8 @@ teardown() {
 @test "build succeeds when requested network is configured" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
     assert_command dfx canister --network tungsten create --all
     assert_command dfx build --network tungsten
 }
@@ -108,7 +109,8 @@ teardown() {
 
 @test "build output for non-local network is in expected directory" {
   dfx_start
-  assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+  webserver_port=$(cat .dfx/webserver-port)
+  assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
   dfx canister --network tungsten create --all
   assert_command dfx build --network tungsten
   assert_command ls .dfx/tungsten/canisters/e2e_project/

e2e/bats/create.bash

@@ -52,7 +52,8 @@ teardown() {
 @test "create succeeds when requested network is configured" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
     assert_command dfx canister --network tungsten create --all
 }
 

e2e/bats/default.nix

@@ -4,47 +4,60 @@
 , use_ic_ref ? false
 }:
 let
-  e2e = lib.noNixFiles (lib.gitOnlySource ./.);
-  lib = pkgs.lib;
-  sources = pkgs.sources;
+  inherit (pkgs) lib;
 
-  inputs = with pkgs; [
-    bats
-    bash
-    coreutils
-    diffutils
-    curl
-    findutils
-    gnugrep
-    gnutar
-    gzip
-    jq
-    netcat
-    ps
-    python3
-    procps
-    which
-    dfx.standalone
-  ] ++ lib.optional use_ic_ref ic-ref;
-in
+  isBatsTest = fileName: type: lib.hasSuffix ".bash" fileName && type == "regular";
 
-builtins.derivation {
-  name = "e2e-tests";
-  system = pkgs.stdenv.system;
-  PATH = pkgs.lib.makeSearchPath "bin" inputs;
-  BATSLIB = sources.bats-support;
-  builder =
-    pkgs.writeScript "builder.sh" ''
-      #!${pkgs.stdenv.shell}
-      set -eo pipefail
+  here = ./.;
 
-      # We want $HOME/.cache to be in a new temporary directory.
-      export HOME=$(mktemp -d -t dfx-e2e-home-XXXX)
+  mkBatsTest = fileName:
+    let
+      name = lib.removeSuffix ".bash" fileName;
+    in
+      lib.nameValuePair name (
+        pkgs.runCommandNoCC "e2e-test-${name}${lib.optionalString use_ic_ref "-use_ic_ref"}" {
+          nativeBuildInputs = with pkgs; [
+            bats
+            diffutils
+            curl
+            findutils
+            gnugrep
+            gnutar
+            gzip
+            jq
+            netcat
+            ps
+            python3
+            procps
+            which
+            dfx.standalone
+          ] ++ lib.optional use_ic_ref ic-ref;
+          BATSLIB = pkgs.sources.bats-support;
+          USE_IC_REF = use_ic_ref;
+          utils = lib.gitOnlySource ./utils;
+          assets = lib.gitOnlySource ./assets;
+          test = here + "/${fileName}";
+        } ''
+          export HOME=$(pwd)
 
-      export USE_IC_REF=${if use_ic_ref then "1" else ""}
+          ln -s $utils utils
+          ln -s $assets assets
+          ln -s $test test
 
-      # Timeout of 10 minutes is enough for now. Reminder; CI might be running with
-      # less resources than a dev's computer, so e2e might take longer.
-      timeout --preserve-status 3600 bats --recursive ${e2e}/* | tee $out
-    '';
-} // { meta = {}; }
+          # Timeout of 10 minutes is enough for now. Reminder; CI might be running with
+          # less resources than a dev's computer, so e2e might take longer.
+          timeout --preserve-status 3600 bats test | tee $out
+        ''
+      );
+in
+builtins.listToAttrs
+  (
+    builtins.map mkBatsTest
+      (
+        lib.attrNames
+          (
+            lib.filterAttrs isBatsTest
+              (builtins.readDir here)
+          )
+      )
+  )

e2e/bats/deploy.bash

@@ -6,16 +6,15 @@ setup() {
     # We want to work from a temporary directory, different for every test.
     cd $(mktemp -d -t dfx-e2e-XXXXXXXX)
     export RUST_BACKTRACE=1
-
-    dfx_new hello
-    dfx_start
 }
 
 teardown() {
   dfx_stop
 }
 
 @test "deploy from a fresh project" {
+    dfx_new hello
+    dfx_start
     install_asset greet
     assert_command dfx deploy
 
@@ -24,32 +23,40 @@ teardown() {
 }
 
 @test "deploy a canister without dependencies" {
+    dfx_new hello
+    dfx_start
     install_asset greet
     assert_command dfx deploy hello
     assert_match 'Deploying: hello'
     assert_not_match 'hello_assets'
 }
 
 @test "deploy a canister with dependencies" {
+    dfx_new hello
+    dfx_start
     install_asset greet
     assert_command dfx deploy hello_assets
     assert_match 'Deploying: hello hello_assets'
 }
 
 @test "deploy a canister with non-circular shared dependencies" {
     install_asset transitive_deps_canisters
+    dfx_start
     assert_command dfx deploy canister_f
     assert_match 'Deploying: canister_a canister_f canister_g canister_h'
 }
 
 @test "report an error on attempt to deploy a canister with circular dependencies" {
     install_asset transitive_deps_canisters
+    dfx_start
     assert_command_fail dfx deploy canister_d
     assert_match 'canister_d -> canister_e -> canister_d'
 }
 
 @test "if already registered, try to upgrade then install" {
+    dfx_new hello
     install_asset greet
+    dfx_start
     assert_command dfx canister create --all
 
     assert_command dfx deploy

e2e/bats/frontend.bash

@@ -19,7 +19,8 @@ teardown() {
     dfx build e2e_project
 
     sleep 1
-    assert_command curl http://localhost:8000 # 8000 = default port.
+    PORT=$(cat .dfx/webserver-port)
+    assert_command curl http://localhost:"$PORT"
     assert_match "<html>"
 }
 
@@ -33,7 +34,7 @@ teardown() {
     dfx canister create --all
     dfx build e2e_project
 
-    assert_command curl http://localhost:12345 # 8000 = default port.
+    assert_command curl http://localhost:12345
     assert_match "<html>"
 
     assert_command_fail curl http://localhost:8000