refactor: enable parallel ci bats tests by enabling dfx start and bootstrap to pick random webserver ports

e2e/bats/bootstrap.bash

@@ -18,10 +18,10 @@ teardown() {
     dfx build
     dfx canister install hello
     ID=$(dfx canister id hello)
-
-    assert_command curl http://localhost:8000/_/candid?canisterId="$ID" -o ./web.txt
+    PORT=$(cat .dfx/webserver-port)
+    assert_command curl http://localhost:"$PORT"/_/candid?canisterId="$ID" -o ./web.txt
     assert_command diff .dfx/local/canisters/hello/hello.did ./web.txt
-    assert_command curl http://localhost:8000/_/candid?canisterId="$ID"\&format=js -o ./web.txt
+    assert_command curl http://localhost:"$PORT"/_/candid?canisterId="$ID"\&format=js -o ./web.txt
     # Relax diff as it's produced by two different compilers.
     assert_command diff --ignore-all-space --ignore-blank-lines .dfx/local/canisters/hello/hello.did.js ./web.txt
 }

e2e/bats/build.bash

@@ -93,7 +93,8 @@ teardown() {
 @test "build succeeds when requested network is configured" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
     assert_command dfx canister --network tungsten create --all
     assert_command dfx build --network tungsten
 }
@@ -108,7 +109,8 @@ teardown() {
 
 @test "build output for non-local network is in expected directory" {
   dfx_start
-  assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+  webserver_port=$(cat .dfx/webserver-port)
+  assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
   dfx canister --network tungsten create --all
   assert_command dfx build --network tungsten
   assert_command ls .dfx/tungsten/canisters/e2e_project/

e2e/bats/create.bash

@@ -52,7 +52,8 @@ teardown() {
 @test "create succeeds when requested network is configured" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
     assert_command dfx canister --network tungsten create --all
 }
 

e2e/bats/default.nix

@@ -4,47 +4,60 @@
 , use_ic_ref ? false
 }:
 let
-  e2e = lib.noNixFiles (lib.gitOnlySource ./.);
-  lib = pkgs.lib;
-  sources = pkgs.sources;
+  inherit (pkgs) lib;
 
-  inputs = with pkgs; [
-    bats
-    bash
-    coreutils
-    diffutils
-    curl
-    findutils
-    gnugrep
-    gnutar
-    gzip
-    jq
-    netcat
-    ps
-    python3
-    procps
-    which
-    dfx.standalone
-  ] ++ lib.optional use_ic_ref ic-ref;
-in
+  isBatsTest = fileName: type: lib.hasSuffix ".bash" fileName && type == "regular";
 
-builtins.derivation {
-  name = "e2e-tests";
-  system = pkgs.stdenv.system;
-  PATH = pkgs.lib.makeSearchPath "bin" inputs;
-  BATSLIB = sources.bats-support;
-  builder =
-    pkgs.writeScript "builder.sh" ''
-      #!${pkgs.stdenv.shell}
-      set -eo pipefail
+  here = ./.;
 
-      # We want $HOME/.cache to be in a new temporary directory.
-      export HOME=$(mktemp -d -t dfx-e2e-home-XXXX)
+  mkBatsTest = fileName:
+    let
+      name = lib.removeSuffix ".bash" fileName;
+    in
+      lib.nameValuePair name (
+        pkgs.runCommandNoCC "e2e-test-${name}${lib.optionalString use_ic_ref "-use_ic_ref"}" {
+          nativeBuildInputs = with pkgs; [
+            bats
+            diffutils
+            curl
+            findutils
+            gnugrep
+            gnutar
+            gzip
+            jq
+            netcat
+            ps
+            python3
+            procps
+            which
+            dfx.standalone
+          ] ++ lib.optional use_ic_ref ic-ref;
+          BATSLIB = pkgs.sources.bats-support;
+          USE_IC_REF = use_ic_ref;
+          utils = lib.gitOnlySource ../../. ./utils;
+          assets = lib.gitOnlySource ../../. ./assets;
+          test = here + "/${fileName}";
+        } ''
+          export HOME=$(pwd)
 
-      export USE_IC_REF=${if use_ic_ref then "1" else ""}
+          ln -s $utils utils
+          ln -s $assets assets
+          ln -s $test test
 
-      # Timeout of 10 minutes is enough for now. Reminder; CI might be running with
-      # less resources than a dev's computer, so e2e might take longer.
-      timeout --preserve-status 3600 bats --recursive ${e2e}/* | tee $out
-    '';
-} // { meta = {}; }
+          # Timeout of 10 minutes is enough for now. Reminder; CI might be running with
+          # less resources than a dev's computer, so e2e might take longer.
+          timeout --preserve-status 3600 bats test | tee $out
+        ''
+      );
+in
+builtins.listToAttrs
+  (
+    builtins.map mkBatsTest
+      (
+        lib.attrNames
+          (
+            lib.filterAttrs isBatsTest
+              (builtins.readDir here)
+          )
+      )
+  )

e2e/bats/frontend.bash

@@ -19,7 +19,8 @@ teardown() {
     dfx build --skip-frontend
 
     sleep 1
-    assert_command curl http://localhost:8000 # 8000 = default port.
+    PORT=$(cat .dfx/webserver-port)
+    assert_command curl http://localhost:"$PORT"
     assert_match "<html>"
 }
 
@@ -33,7 +34,7 @@ teardown() {
     dfx canister create --all
     dfx build --skip-frontend
 
-    assert_command curl http://localhost:12345 # 8000 = default port.
+    assert_command curl http://localhost:12345
     assert_match "<html>"
 
     assert_command_fail curl http://localhost:8000

e2e/bats/network.bash

@@ -17,7 +17,8 @@ teardown() {
 @test "create stores canister ids for default-persistent networks in canister_ids.json" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
 
     assert_command dfx canister --network tungsten create --all
 
@@ -29,7 +30,8 @@ teardown() {
 @test "create stores canister ids for configured-ephemeral networks in canister_ids.json" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
     cat <<<$(jq .networks.tungsten.type=\"ephemeral\" dfx.json) >dfx.json
 
     assert_command dfx canister --network tungsten create --all
@@ -71,7 +73,8 @@ teardown() {
 @test "failure message does include network if for non-local network" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
 
     assert_command_fail dfx build --network tungsten
     assert_match "Cannot find canister id. Please issue 'dfx canister --network tungsten create e2e_project"

e2e/bats/ping.bash

@@ -17,9 +17,6 @@ teardown() {
     assert_command_fail dfx ping
 }
 
-@test "dfx start succeeds" {
-    dfx_start
-}
 
 @test "dfx ping succeeds if replica is running" {
     dfx_start
@@ -30,7 +27,8 @@ teardown() {
 
 @test "dfx ping succeeds by specific host:post" {
     dfx_start
-    assert_command dfx ping http://127.0.0.1:8000
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx ping http://127.0.0.1:"$webserver_port"
 
     assert_match "\"ic_api_version\""
 }
@@ -44,7 +42,8 @@ teardown() {
 
 @test "dfx ping succeeds by network name if network bind address is host:port format" {
     dfx_start
-    assert_command dfx config networks.local.bind '"127.0.0.1:8000"'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.local.bind '"127.0.0.1:'$webserver_port'"'
     assert_command dfx ping local
 
     assert_match "\"ic_api_version\""