From 0964da9476ff29cf1aadb0d120a7ffdb7f8669f5 Mon Sep 17 00:00:00 2001
From: Bas van Dijk <v.dijk.bas@gmail.com>
Date: Mon, 27 Jul 2020 22:39:02 +0200
Subject: [PATCH 01/19] build: parallelize the e2e bats tests

Running the e2e bats test sequentially took 15m on my laptop. We
should run them all in parallel such that we don't have to wait so
long on CI.
---
 e2e/bats/default.nix | 89 +++++++++++++++++++++++++-------------------
 1 file changed, 50 insertions(+), 39 deletions(-)

diff --git a/e2e/bats/default.nix b/e2e/bats/default.nix
index f4f0e92b73..15fa30f35a 100644
--- a/e2e/bats/default.nix
+++ b/e2e/bats/default.nix
@@ -4,47 +4,58 @@
 , use_ic_ref ? false
 }:
 let
-  e2e = lib.noNixFiles (lib.gitOnlySource ./.);
-  lib = pkgs.lib;
-  sources = pkgs.sources;
+  inherit (pkgs) lib;
 
-  inputs = with pkgs; [
-    bats
-    bash
-    coreutils
-    diffutils
-    curl
-    findutils
-    gnugrep
-    gnutar
-    gzip
-    jq
-    netcat
-    ps
-    python3
-    procps
-    which
-    dfx.standalone
-  ] ++ lib.optional use_ic_ref ic-ref;
-in
+  isBatsTest = fileName: type: lib.hasSuffix ".bash" fileName && type == "regular";
 
-builtins.derivation {
-  name = "e2e-tests";
-  system = pkgs.stdenv.system;
-  PATH = pkgs.lib.makeSearchPath "bin" inputs;
-  BATSLIB = sources.bats-support;
-  builder =
-    pkgs.writeScript "builder.sh" ''
-      #!${pkgs.stdenv.shell}
-      set -eo pipefail
+  here = ./.;
 
-      # We want $HOME/.cache to be in a new temporary directory.
-      export HOME=$(mktemp -d -t dfx-e2e-home-XXXX)
+  mkBatsTest = fileName:
+    let
+      name = lib.removeSuffix ".bash" fileName;
+    in
+      lib.nameValuePair name (
+        pkgs.runCommandNoCC name {
+          nativeBuildInputs = with pkgs; [
+            bats
+            diffutils
+            curl
+            findutils
+            gnugrep
+            gnutar
+            gzip
+            jq
+            netcat
+            ps
+            python3
+            procps
+            which
+            dfx.standalone
+          ] ++ lib.optional use_ic_ref ic-ref;
+          BATSLIB = pkgs.sources.bats-support;
+          USE_IC_REF = use_ic_ref;
+        } ''
+          # We want $HOME/.cache to be in a new temporary directory.
+          export HOME=$(mktemp -d -t dfx-e2e-home-XXXX)
 
-      export USE_IC_REF=${if use_ic_ref then "1" else ""}
+          ln -s ${./utils} utils
+          ln -s ${./assets} assets
+          ln -s ${here + "/${fileName}"} ${fileName} 
 
-      # Timeout of 10 minutes is enough for now. Reminder; CI might be running with
-      # less resources than a dev's computer, so e2e might take longer.
-      timeout --preserve-status 3600 bats --recursive ${e2e}/* | tee $out
-    '';
-} // { meta = {}; }
+          # Timeout of 10 minutes is enough for now. Reminder; CI might be running with
+          # less resources than a dev's computer, so e2e might take longer.
+          timeout --preserve-status 3600 bats ${fileName} | tee $out
+        ''
+      );
+in
+builtins.listToAttrs
+  (
+    builtins.map mkBatsTest
+      (
+        lib.attrNames
+          (
+            lib.filterAttrs isBatsTest
+              (builtins.readDir here)
+          )
+      )
+  )

From d205c44ecb2cc1354adfdec8b72f54ca29e3932c Mon Sep 17 00:00:00 2001
From: Bas van Dijk <v.dijk.bas@gmail.com>
Date: Tue, 28 Jul 2020 09:31:18 +0200
Subject: [PATCH 02/19] e2e/bats: better name of the derivations

---
 e2e/bats/default.nix | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/e2e/bats/default.nix b/e2e/bats/default.nix
index 15fa30f35a..b468eebae4 100644
--- a/e2e/bats/default.nix
+++ b/e2e/bats/default.nix
@@ -15,7 +15,7 @@ let
       name = lib.removeSuffix ".bash" fileName;
     in
       lib.nameValuePair name (
-        pkgs.runCommandNoCC name {
+        pkgs.runCommandNoCC "e2e-test-${name}${lib.optionalString use_ic_ref "-use_ic_ref"}" {
           nativeBuildInputs = with pkgs; [
             bats
             diffutils
@@ -34,17 +34,19 @@ let
           ] ++ lib.optional use_ic_ref ic-ref;
           BATSLIB = pkgs.sources.bats-support;
           USE_IC_REF = use_ic_ref;
+          utils = lib.gitOnlySource ../../. ./utils;
+          assets = lib.gitOnlySource ../../. ./assets;
+          test = here + "/${fileName}";
         } ''
-          # We want $HOME/.cache to be in a new temporary directory.
-          export HOME=$(mktemp -d -t dfx-e2e-home-XXXX)
+          export HOME=$(pwd)
 
-          ln -s ${./utils} utils
-          ln -s ${./assets} assets
-          ln -s ${here + "/${fileName}"} ${fileName} 
+          ln -s $utils utils
+          ln -s $assets assets
+          ln -s $test test
 
           # Timeout of 10 minutes is enough for now. Reminder; CI might be running with
           # less resources than a dev's computer, so e2e might take longer.
-          timeout --preserve-status 3600 bats ${fileName} | tee $out
+          timeout --preserve-status 3600 bats test | tee $out
         ''
       );
 in

From a786ad07c52cfc31dfb6cf503d7adcb2f53aea0f Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Tue, 11 Aug 2020 18:30:15 -0700
Subject: [PATCH 03/19] add stuff

---
 Cargo.lock                        |  1 +
 e2e/bats/bootstrap.bash           |  6 +++---
 e2e/bats/build.bash               |  6 ++++--
 e2e/bats/create.bash              |  3 ++-
 e2e/bats/frontend.bash            |  5 +++--
 e2e/bats/network.bash             |  9 ++++++---
 e2e/bats/ping.bash                |  9 ++++-----
 e2e/bats/utils/_.bash             | 14 ++++++++++++--
 src/dfx/Cargo.toml                |  1 +
 src/dfx/src/commands/bootstrap.rs | 15 +++++++++++++--
 src/dfx/src/commands/start.rs     | 32 ++++++++++++++++++++++---------
 src/dfx/src/util/mod.rs           | 20 +++++++++++++++++++
 12 files changed, 92 insertions(+), 29 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ce9ac67a95..c2bd80e3b9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1018,6 +1018,7 @@ dependencies = [
  "libflate",
  "mockall",
  "mockito",
+ "net2",
  "petgraph",
  "proptest",
  "rand 0.7.3",
diff --git a/e2e/bats/bootstrap.bash b/e2e/bats/bootstrap.bash
index 8a08648aac..0d1df0343c 100644
--- a/e2e/bats/bootstrap.bash
+++ b/e2e/bats/bootstrap.bash
@@ -18,10 +18,10 @@ teardown() {
     dfx build
     dfx canister install hello
     ID=$(dfx canister id hello)
-
-    assert_command curl http://localhost:8000/_/candid?canisterId="$ID" -o ./web.txt
+    PORT=$(cat .dfx/webserver-port)
+    assert_command curl http://localhost:"$PORT"/_/candid?canisterId="$ID" -o ./web.txt
     assert_command diff .dfx/local/canisters/hello/hello.did ./web.txt
-    assert_command curl http://localhost:8000/_/candid?canisterId="$ID"\&format=js -o ./web.txt
+    assert_command curl http://localhost:"$PORT"/_/candid?canisterId="$ID"\&format=js -o ./web.txt
     # Relax diff as it's produced by two different compilers.
     assert_command diff --ignore-all-space --ignore-blank-lines .dfx/local/canisters/hello/hello.did.js ./web.txt
 }
diff --git a/e2e/bats/build.bash b/e2e/bats/build.bash
index c6a8676897..15b4025887 100644
--- a/e2e/bats/build.bash
+++ b/e2e/bats/build.bash
@@ -93,7 +93,8 @@ teardown() {
 @test "build succeeds when requested network is configured" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
     assert_command dfx canister --network tungsten create --all
     assert_command dfx build --network tungsten
 }
@@ -108,7 +109,8 @@ teardown() {
 
 @test "build output for non-local network is in expected directory" {
   dfx_start
-  assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+  webserver_port=$(cat .dfx/webserver-port)
+  assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
   dfx canister --network tungsten create --all
   assert_command dfx build --network tungsten
   assert_command ls .dfx/tungsten/canisters/e2e_project/
diff --git a/e2e/bats/create.bash b/e2e/bats/create.bash
index 00c289ec78..b7004bce75 100644
--- a/e2e/bats/create.bash
+++ b/e2e/bats/create.bash
@@ -52,7 +52,8 @@ teardown() {
 @test "create succeeds when requested network is configured" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
     assert_command dfx canister --network tungsten create --all
 }
 
diff --git a/e2e/bats/frontend.bash b/e2e/bats/frontend.bash
index c242347596..31cda2626a 100644
--- a/e2e/bats/frontend.bash
+++ b/e2e/bats/frontend.bash
@@ -19,7 +19,8 @@ teardown() {
     dfx build --skip-frontend
 
     sleep 1
-    assert_command curl http://localhost:8000 # 8000 = default port.
+    PORT=$(cat .dfx/webserver-port)
+    assert_command curl http://localhost:"$PORT"
     assert_match "<html>"
 }
 
@@ -33,7 +34,7 @@ teardown() {
     dfx canister create --all
     dfx build --skip-frontend
 
-    assert_command curl http://localhost:12345 # 8000 = default port.
+    assert_command curl http://localhost:12345
     assert_match "<html>"
 
     assert_command_fail curl http://localhost:8000
diff --git a/e2e/bats/network.bash b/e2e/bats/network.bash
index 9a3abe0f6e..eec57d13cc 100644
--- a/e2e/bats/network.bash
+++ b/e2e/bats/network.bash
@@ -17,7 +17,8 @@ teardown() {
 @test "create stores canister ids for default-persistent networks in canister_ids.json" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
 
     assert_command dfx canister --network tungsten create --all
 
@@ -29,7 +30,8 @@ teardown() {
 @test "create stores canister ids for configured-ephemeral networks in canister_ids.json" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
     cat <<<$(jq .networks.tungsten.type=\"ephemeral\" dfx.json) >dfx.json
 
     assert_command dfx canister --network tungsten create --all
@@ -71,7 +73,8 @@ teardown() {
 @test "failure message does include network if for non-local network" {
     dfx_start
 
-    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:8000" ]'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.tungsten.providers '[ "http://127.0.0.1:'$webserver_port'" ]'
 
     assert_command_fail dfx build --network tungsten
     assert_match "Cannot find canister id. Please issue 'dfx canister --network tungsten create e2e_project"
diff --git a/e2e/bats/ping.bash b/e2e/bats/ping.bash
index 62af13b3f5..44d6cb0530 100644
--- a/e2e/bats/ping.bash
+++ b/e2e/bats/ping.bash
@@ -17,9 +17,6 @@ teardown() {
     assert_command_fail dfx ping
 }
 
-@test "dfx start succeeds" {
-    dfx_start
-}
 
 @test "dfx ping succeeds if replica is running" {
     dfx_start
@@ -30,7 +27,8 @@ teardown() {
 
 @test "dfx ping succeeds by specific host:post" {
     dfx_start
-    assert_command dfx ping http://127.0.0.1:8000
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx ping http://127.0.0.1:"$webserver_port"
 
     assert_match "\"ic_api_version\""
 }
@@ -44,7 +42,8 @@ teardown() {
 
 @test "dfx ping succeeds by network name if network bind address is host:port format" {
     dfx_start
-    assert_command dfx config networks.local.bind '"127.0.0.1:8000"'
+    webserver_port=$(cat .dfx/webserver-port)
+    assert_command dfx config networks.local.bind '"127.0.0.1:'$webserver_port'"'
     assert_command dfx ping local
 
     assert_match "\"ic_api_version\""
diff --git a/e2e/bats/utils/_.bash b/e2e/bats/utils/_.bash
index 003e9c3cd1..833b3a5fe3 100644
--- a/e2e/bats/utils/_.bash
+++ b/e2e/bats/utils/_.bash
@@ -43,21 +43,31 @@ dfx_start() {
 
         cat <<<$(jq .networks.local.bind=\"127.0.0.1:${port}\" dfx.json) >dfx.json
         cat dfx.json
-        dfx bootstrap --port 8000 &
+        dfx bootstrap --port 0 & # Start on random port for parallel test execution
+        local webserver_port=$(cat .dfx/webserver-port)
         echo $! > dfx-bootstrap.pid
     else
         # Bats creates a FD 3 for test output, but child processes inherit it and Bats will
         # wait for it to close. Because `dfx start` leaves child processes running, we need
         # to close this pipe, otherwise Bats will wait indefinitely.
-        dfx start --background "$@" 3>&-
+        if [[ "$@" == "" ]]; then
+            dfx start --background --host "127.0.0.1:0" 3>&- # Start on random port for parallel test execution
+        else
+            dfx start --background "$@" 3>&- # Start on random port for parallel test execution
+        fi
         local project_dir=${pwd}
         local dfx_config_root=.dfx/client-configuration
         printf "Configuration Root for DFX: %s\n" "${dfx_config_root}"
         test -f ${dfx_config_root}/client-1.port
         local port=$(cat ${dfx_config_root}/client-1.port)
+
+        # Overwrite the default networks.local.bind 127.0.0.1:8000 with allocated port
+        local webserver_port=$(cat .dfx/webserver-port)
+        cat <<<$(jq .networks.local.bind=\"127.0.0.1:${webserver_port}\" dfx.json) >dfx.json
     fi
 
     printf "Replica Configured Port: %s\n" "${port}"
+    printf "Webserver Configured Port: %s\n" "${webserver_port}"
 
     timeout 5 sh -c \
         "until nc -z localhost ${port}; do echo waiting for replica; sleep 1; done" \
diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml
index 8fd13fac1d..33815b59b9 100644
--- a/src/dfx/Cargo.toml
+++ b/src/dfx/Cargo.toml
@@ -39,6 +39,7 @@ lazy_static = "1.4.0"
 libflate = "0.1.27"
 hotwatch = "0.4.3"
 mockall = "0.6.0"
+net2 = "0.2.34"
 petgraph = "0.5.0"
 rand = "0.7.2"
 regex = "1.3.1"
diff --git a/src/dfx/src/commands/bootstrap.rs b/src/dfx/src/commands/bootstrap.rs
index e5edff1c23..3c969a4735 100644
--- a/src/dfx/src/commands/bootstrap.rs
+++ b/src/dfx/src/commands/bootstrap.rs
@@ -5,12 +5,13 @@ use crate::lib::message::UserMessage;
 use crate::lib::network::network_descriptor::NetworkDescriptor;
 use crate::lib::provider::get_network_descriptor;
 use crate::lib::webserver::webserver;
+use crate::util::get_reusable_socket_addr;
 use clap::{App, Arg, ArgMatches, SubCommand};
 use slog::info;
 use std::default::Default;
 use std::fs;
 use std::io::{Error, ErrorKind};
-use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::net::{IpAddr, Ipv4Addr};
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::time::Duration;
@@ -70,11 +71,21 @@ pub fn exec(env: &dyn Environment, args: &ArgMatches<'_>) -> DfxResult {
 
     let (sender, receiver) = crossbeam::unbounded();
 
+    // Since the user may have provided port "0", we need to grab a dynamically
+    // allocated port and construct a resuable SocketAddr which the actix
+    // HttpServer will bind to
+    let socket_addr =
+        get_reusable_socket_addr(config_bootstrap.ip.unwrap(), config_bootstrap.port.unwrap())?;
+
+    let webserver_port_path = env.get_temp_dir().join("webserver-port");
+    std::fs::write(&webserver_port_path, "")?;
+    std::fs::write(&webserver_port_path, socket_addr.port().to_string())?;
+
     webserver(
         logger.clone(),
         build_output_root,
         network_descriptor,
-        SocketAddr::new(config_bootstrap.ip.unwrap(), config_bootstrap.port.unwrap()),
+        socket_addr,
         providers
             .iter()
             .map(|uri| Url::from_str(uri).unwrap())
diff --git a/src/dfx/src/commands/start.rs b/src/dfx/src/commands/start.rs
index affe6d54c5..59d7ca2df8 100644
--- a/src/dfx/src/commands/start.rs
+++ b/src/dfx/src/commands/start.rs
@@ -6,6 +6,7 @@ use crate::lib::provider::get_network_descriptor;
 use crate::lib::proxy::{CoordinateProxy, ProxyConfig};
 use crate::lib::proxy_process::spawn_and_update_proxy;
 use crate::lib::replica_config::ReplicaConfig;
+use crate::util::get_reusable_socket_addr;
 
 use clap::{App, Arg, ArgMatches, SubCommand};
 use crossbeam::channel::{Receiver, Sender};
@@ -19,9 +20,9 @@ use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::process::Command;
 use std::time::Duration;
+
 use sysinfo::{Pid, Process, ProcessExt, Signal, System, SystemExt};
 use tokio::runtime::Runtime;
-
 /// Provide necessary arguments to start the Internet Computer
 /// locally. See `exec` for further information.
 pub fn construct() -> App<'static, 'static> {
@@ -75,12 +76,16 @@ pub fn exec(env: &dyn Environment, args: &ArgMatches<'_>) -> DfxResult {
         .get_config()
         .ok_or(DfxError::CommandMustBeRunInAProject)?;
 
+    let temp_dir = env.get_temp_dir();
+
     let (frontend_url, address_and_port) = frontend_address(args, &config)?;
+    let webserver_port_path = temp_dir.join("webserver-port");
+    std::fs::write(&webserver_port_path, "")?;
+    std::fs::write(&webserver_port_path, address_and_port.port().to_string())?;
 
     let client_pathbuf = env.get_cache().get_binary_command_path("replica")?;
     let ic_starter_pathbuf = env.get_cache().get_binary_command_path("ic-starter")?;
 
-    let temp_dir = env.get_temp_dir();
     let state_root = env.get_state_dir();
 
     let pid_file_path = temp_dir.join("pid");
@@ -117,7 +122,7 @@ pub fn exec(env: &dyn Environment, args: &ArgMatches<'_>) -> DfxResult {
 
     if args.is_present("background") {
         send_background()?;
-        return ping_and_wait(&frontend_url);
+        return Ok(());
     }
 
     // Start the client.
@@ -284,7 +289,7 @@ fn send_background() -> DfxResult<()> {
 }
 
 fn frontend_address(args: &ArgMatches<'_>, config: &Config) -> DfxResult<(String, SocketAddr)> {
-    let address_and_port = args
+    let mut address_and_port = args
         .value_of("host")
         .and_then(|host| Option::from(host.parse()))
         .unwrap_or_else(|| {
@@ -294,12 +299,21 @@ fn frontend_address(args: &ArgMatches<'_>, config: &Config) -> DfxResult<(String
                 .expect("could not get socket_addr"))
         })
         .map_err(|e| DfxError::InvalidArgument(format!("Invalid host: {}", e)))?;
-    let frontend_url = format!(
-        "http://{}:{}",
-        address_and_port.ip(),
-        address_and_port.port()
-    );
 
+    if !args.is_present("background") {
+        // Since the user may have provided port "0", we need to grab a dynamically
+        // allocated port and construct a resuable SocketAddr which the actix
+        // HttpServer will bind to
+        address_and_port =
+            get_reusable_socket_addr(address_and_port.ip(), address_and_port.port())?;
+    }
+    let ip = if address_and_port.is_ipv6() {
+        format!("[{}]", address_and_port.ip())
+    } else {
+        address_and_port.ip().to_string()
+    };
+    let frontend_url = format!("http://{}:{}", ip, address_and_port.port());
+    println!("frontend_url {:?}", frontend_url);
     Ok((frontend_url, address_and_port))
 }
 
diff --git a/src/dfx/src/util/mod.rs b/src/dfx/src/util/mod.rs
index c55918f2e2..ea891022ef 100644
--- a/src/dfx/src/util/mod.rs
+++ b/src/dfx/src/util/mod.rs
@@ -3,10 +3,30 @@ use candid::parser::typing::{check_prog, TypeEnv};
 use candid::types::{Function, Type};
 use candid::{parser::value::IDLValue, IDLArgs, IDLProg};
 use ic_agent::Blob;
+use net2::{unix::UnixTcpBuilderExt, TcpBuilder};
+use std::net::{IpAddr, SocketAddr};
 
 pub mod assets;
 pub mod clap;
 
+// The user can pass in port "0" to dfx start or dfx bootstrap i.e. "127.0.0.1:0" or "[::1]:0",
+// thus, we need to recreate SocketAddr with the kernel provided dynmically allocated port here.
+// TcpBuilder is used with reuse_address and reuse_port set to "true" because
+// the Actix HttpServer in webserver.rs will bind to this SocketAddr.
+pub fn get_reusable_socket_addr(ip: IpAddr, port: u16) -> DfxResult<SocketAddr> {
+    let tcp_builder = if ip.is_ipv4() {
+        TcpBuilder::new_v4()?
+    } else {
+        TcpBuilder::new_v6()?
+    };
+    Ok(tcp_builder
+        .bind(SocketAddr::new(ip, port))?
+        .reuse_address(true)?
+        .reuse_port(true)?
+        .to_tcp_listener()?
+        .local_addr()?)
+}
+
 /// Deserialize and print return values from canister method.
 pub fn print_idl_blob(
     blob: &Blob,

From 3b8d06122f7a9578390e04a3779a1178f149af38 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Wed, 12 Aug 2020 16:36:09 -0700
Subject: [PATCH 04/19] add sleep

---
 e2e/bats/utils/_.bash | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/e2e/bats/utils/_.bash b/e2e/bats/utils/_.bash
index 833b3a5fe3..35ee2306fa 100644
--- a/e2e/bats/utils/_.bash
+++ b/e2e/bats/utils/_.bash
@@ -53,12 +53,13 @@ dfx_start() {
         if [[ "$@" == "" ]]; then
             dfx start --background --host "127.0.0.1:0" 3>&- # Start on random port for parallel test execution
         else
-            dfx start --background "$@" 3>&- # Start on random port for parallel test execution
+            dfx start --background "$@" 3>&-
         fi
         local project_dir=${pwd}
         local dfx_config_root=.dfx/client-configuration
         printf "Configuration Root for DFX: %s\n" "${dfx_config_root}"
         test -f ${dfx_config_root}/client-1.port
+        sleep 3
         local port=$(cat ${dfx_config_root}/client-1.port)
 
         # Overwrite the default networks.local.bind 127.0.0.1:8000 with allocated port

From d0938a890312771439559f187006f6c1f50ed204 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Wed, 12 Aug 2020 18:24:09 -0700
Subject: [PATCH 05/19] custom asset canister uses localhost 8000 so start
 command should do the same

---
 e2e/bats/build.bash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/e2e/bats/build.bash b/e2e/bats/build.bash
index 15b4025887..1672d20ee8 100644
--- a/e2e/bats/build.bash
+++ b/e2e/bats/build.bash
@@ -74,7 +74,7 @@ teardown() {
 }
 
 @test "can build a custom canister type" {
-  dfx_start
+  dfx_start --host 127.0.0.1:8000
   install_asset custom_canister
   dfx canister create --all
   assert_command dfx build

From e28c932e2c08c6af71234e680b04468f66244748 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Wed, 12 Aug 2020 19:03:46 -0700
Subject: [PATCH 06/19] ic ref fix

---
 e2e/bats/utils/_.bash | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/e2e/bats/utils/_.bash b/e2e/bats/utils/_.bash
index 35ee2306fa..56ec443d9e 100644
--- a/e2e/bats/utils/_.bash
+++ b/e2e/bats/utils/_.bash
@@ -43,7 +43,11 @@ dfx_start() {
 
         cat <<<$(jq .networks.local.bind=\"127.0.0.1:${port}\" dfx.json) >dfx.json
         cat dfx.json
-        dfx bootstrap --port 0 & # Start on random port for parallel test execution
+        if [[ "$@" == "" ]]; then
+            dfx bootstrap --port 0 & # Start on random port for parallel test execution
+        else
+            dfx bootstrap --port "$@" &
+        fi
         local webserver_port=$(cat .dfx/webserver-port)
         echo $! > dfx-bootstrap.pid
     else

From f303e3625bad31570fdaf343d872cb0108cd8495 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Wed, 12 Aug 2020 21:09:56 -0700
Subject: [PATCH 07/19] port should be default

---
 e2e/bats/utils/_.bash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/e2e/bats/utils/_.bash b/e2e/bats/utils/_.bash
index 56ec443d9e..d68b81e6ce 100644
--- a/e2e/bats/utils/_.bash
+++ b/e2e/bats/utils/_.bash
@@ -46,7 +46,7 @@ dfx_start() {
         if [[ "$@" == "" ]]; then
             dfx bootstrap --port 0 & # Start on random port for parallel test execution
         else
-            dfx bootstrap --port "$@" &
+            dfx bootstrap --port 8000 &
         fi
         local webserver_port=$(cat .dfx/webserver-port)
         echo $! > dfx-bootstrap.pid

From 15b7f9c7c2522af512d94536355ca34f7476b767 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Mon, 24 Aug 2020 14:02:26 -0700
Subject: [PATCH 08/19] try suggestion

---
 e2e/bats/default.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/e2e/bats/default.nix b/e2e/bats/default.nix
index b468eebae4..8f5ff7487d 100644
--- a/e2e/bats/default.nix
+++ b/e2e/bats/default.nix
@@ -34,8 +34,8 @@ let
           ] ++ lib.optional use_ic_ref ic-ref;
           BATSLIB = pkgs.sources.bats-support;
           USE_IC_REF = use_ic_ref;
-          utils = lib.gitOnlySource ../../. ./utils;
-          assets = lib.gitOnlySource ../../. ./assets;
+          utils = lib.gitOnlySource ./utils;
+          assets = lib.gitOnlySource ./assets;
           test = here + "/${fileName}";
         } ''
           export HOME=$(pwd)

From 515c949eb62ecc53225adab8b856ef5f1600135c Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Mon, 24 Aug 2020 14:25:33 -0700
Subject: [PATCH 09/19] increase sleep to reduce flake

---
 e2e/bats/utils/_.bash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/e2e/bats/utils/_.bash b/e2e/bats/utils/_.bash
index d68b81e6ce..fcbf0a67c4 100644
--- a/e2e/bats/utils/_.bash
+++ b/e2e/bats/utils/_.bash
@@ -63,7 +63,7 @@ dfx_start() {
         local dfx_config_root=.dfx/client-configuration
         printf "Configuration Root for DFX: %s\n" "${dfx_config_root}"
         test -f ${dfx_config_root}/client-1.port
-        sleep 3
+        sleep 5
         local port=$(cat ${dfx_config_root}/client-1.port)
 
         # Overwrite the default networks.local.bind 127.0.0.1:8000 with allocated port

From 255d655080f54dfb69d5d748572aa08296d97584 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Mon, 24 Aug 2020 16:06:44 -0700
Subject: [PATCH 10/19] increase ping_and_wait wait to 5 secs

---
 src/dfx/src/commands/start.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/dfx/src/commands/start.rs b/src/dfx/src/commands/start.rs
index 59d7ca2df8..c3b44c28bf 100644
--- a/src/dfx/src/commands/start.rs
+++ b/src/dfx/src/commands/start.rs
@@ -58,7 +58,7 @@ fn ping_and_wait(frontend_url: &str) -> DfxResult {
 
     // wait for frontend to come up
     use std::{thread, time};
-    let three_secs = time::Duration::from_secs(3);
+    let three_secs = time::Duration::from_secs(5);
     thread::sleep(three_secs);
 
     runtime

From 3342adbbdf1bc0c2305422cfb6d4cbda68239c37 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Mon, 28 Sep 2020 18:08:15 -0700
Subject: [PATCH 11/19] fix deploy and set write permissions when installing
 assets

---
 e2e/bats/deploy.bash  | 13 ++++++++++---
 e2e/bats/utils/_.bash |  2 ++
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/e2e/bats/deploy.bash b/e2e/bats/deploy.bash
index 865a2d0273..3ae6263c67 100644
--- a/e2e/bats/deploy.bash
+++ b/e2e/bats/deploy.bash
@@ -6,9 +6,6 @@ setup() {
     # We want to work from a temporary directory, different for every test.
     cd $(mktemp -d -t dfx-e2e-XXXXXXXX)
     export RUST_BACKTRACE=1
-
-    dfx_new hello
-    dfx_start
 }
 
 teardown() {
@@ -16,6 +13,8 @@ teardown() {
 }
 
 @test "deploy from a fresh project" {
+    dfx_new hello
+    dfx_start
     install_asset greet
     assert_command dfx deploy
 
@@ -24,6 +23,8 @@ teardown() {
 }
 
 @test "deploy a canister without dependencies" {
+    dfx_new hello
+    dfx_start
     install_asset greet
     assert_command dfx deploy hello
     assert_match 'Deploying: hello'
@@ -31,6 +32,8 @@ teardown() {
 }
 
 @test "deploy a canister with dependencies" {
+    dfx_new hello
+    dfx_start
     install_asset greet
     assert_command dfx deploy hello_assets
     assert_match 'Deploying: hello hello_assets'
@@ -38,18 +41,22 @@ teardown() {
 
 @test "deploy a canister with non-circular shared dependencies" {
     install_asset transitive_deps_canisters
+    dfx_start
     assert_command dfx deploy canister_f
     assert_match 'Deploying: canister_a canister_f canister_g canister_h'
 }
 
 @test "report an error on attempt to deploy a canister with circular dependencies" {
     install_asset transitive_deps_canisters
+    dfx_start
     assert_command_fail dfx deploy canister_d
     assert_match 'canister_d -> canister_e -> canister_d'
 }
 
 @test "if already registered, try to upgrade then install" {
+    dfx_new hello
     install_asset greet
+    dfx_start
     assert_command dfx canister create --all
 
     assert_command dfx deploy
diff --git a/e2e/bats/utils/_.bash b/e2e/bats/utils/_.bash
index fcbf0a67c4..1505a33083 100644
--- a/e2e/bats/utils/_.bash
+++ b/e2e/bats/utils/_.bash
@@ -5,6 +5,8 @@ load utils/assertions
 install_asset() {
     ASSET_ROOT=${BATS_TEST_DIRNAME}/assets/$1/
     cp -R $ASSET_ROOT/* .
+    # set write perms to overwrite local bind in assets which have a dfx.json
+    chmod -R a+w .
 
     [ -f ./patch.bash ] && source ./patch.bash
 }

From 6bde15ae596dcea83c20188852401d0318efaa55 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Mon, 28 Sep 2020 18:30:19 -0700
Subject: [PATCH 12/19] fix ping.bash

---
 e2e/bats/ping.bash | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/e2e/bats/ping.bash b/e2e/bats/ping.bash
index 44d6cb0530..2e25b8a52a 100644
--- a/e2e/bats/ping.bash
+++ b/e2e/bats/ping.bash
@@ -53,6 +53,8 @@ teardown() {
     [ "$USE_IC_REF" ] && skip "skipped for ic-ref"
 
     dfx_start --host 127.0.0.1:12345
+    # dfx_start overwrites local bind with provided port arg, set it back to default
+    assert_command dfx config networks.local.bind '"127.0.0.1:8000"'
     cat <<<$(jq .networks.arbitrary.providers=[\"http://127.0.0.1:12345\"] dfx.json) >dfx.json
 
     assert_command dfx ping arbitrary

From c4d9e1a62c1f08d384b13921d60bacd8615b455c Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Mon, 28 Sep 2020 18:34:23 -0700
Subject: [PATCH 13/19] make custom_canister assets port dynamic

---
 e2e/bats/build.bash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/e2e/bats/build.bash b/e2e/bats/build.bash
index 1672d20ee8..89718f97a7 100644
--- a/e2e/bats/build.bash
+++ b/e2e/bats/build.bash
@@ -74,8 +74,8 @@ teardown() {
 }
 
 @test "can build a custom canister type" {
-  dfx_start --host 127.0.0.1:8000
   install_asset custom_canister
+  dfx_start
   dfx canister create --all
   assert_command dfx build
   assert_match "CUSTOM_CANISTER_BUILD_DONE"

From a594b0b00975a72997d3fb2ef73c3f99d15bcc06 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Mon, 28 Sep 2020 18:35:54 -0700
Subject: [PATCH 14/19] remove debug print

---
 src/dfx/src/commands/start.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/dfx/src/commands/start.rs b/src/dfx/src/commands/start.rs
index dfc45b73dd..cd578676ee 100644
--- a/src/dfx/src/commands/start.rs
+++ b/src/dfx/src/commands/start.rs
@@ -314,7 +314,6 @@ fn frontend_address(args: &ArgMatches<'_>, config: &Config) -> DfxResult<(String
         address_and_port.ip().to_string()
     };
     let frontend_url = format!("http://{}:{}", ip, address_and_port.port());
-    println!("frontend_url {:?}", frontend_url);
     Ok((frontend_url, address_and_port))
 }
 

From 8d6b073214ad8a133ef3864ca1c8d65192cbe599 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Mon, 28 Sep 2020 18:37:45 -0700
Subject: [PATCH 15/19] readd comment

---
 src/dfx/src/commands/start.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/dfx/src/commands/start.rs b/src/dfx/src/commands/start.rs
index cd578676ee..05a8fee3d1 100644
--- a/src/dfx/src/commands/start.rs
+++ b/src/dfx/src/commands/start.rs
@@ -21,9 +21,9 @@ use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::process::Command;
 use std::time::Duration;
-
 use sysinfo::{Pid, Process, ProcessExt, Signal, System, SystemExt};
 use tokio::runtime::Runtime;
+
 /// Provide necessary arguments to start the Internet Computer
 /// locally. See `exec` for further information.
 pub fn construct() -> App<'static, 'static> {
@@ -54,6 +54,7 @@ fn ping_and_wait(frontend_url: &str) -> DfxResult {
 
     let agent = Agent::builder().with_url(frontend_url).build()?;
 
+    // wait for frontend to come up
     let mut waiter = Delay::builder()
         .timeout(std::time::Duration::from_secs(30))
         .throttle(std::time::Duration::from_secs(1))

From 07dd559c8423b42d7d20f716576cb12a002acfc4 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Tue, 29 Sep 2020 12:08:43 -0700
Subject: [PATCH 16/19] add fg_ping_and_wait where the fg process verifys
 frontend is up before exiting

---
 src/dfx/Cargo.toml            |  2 +-
 src/dfx/src/commands/start.rs | 44 ++++++++++++++++++++++++++++++++---
 src/dfx/src/lib/error/mod.rs  | 16 +++++++++++++
 3 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml
index 8634f726eb..62a320211b 100644
--- a/src/dfx/Cargo.toml
+++ b/src/dfx/Cargo.toml
@@ -62,7 +62,7 @@ tar = "0.4.26"
 tempfile = "3.1.0"
 thiserror = "1.0.20"
 toml = "0.5.5"
-tokio = "0.2.10"
+tokio = { version = "0.2.10", features = [ "fs" ] }
 url = "2.1.0"
 walkdir = "2.2.9"
 wasmparser = "0.45.0"
diff --git a/src/dfx/src/commands/start.rs b/src/dfx/src/commands/start.rs
index 05a8fee3d1..05ac732c98 100644
--- a/src/dfx/src/commands/start.rs
+++ b/src/dfx/src/commands/start.rs
@@ -16,7 +16,7 @@ use futures::executor::block_on;
 use ic_agent::Agent;
 use indicatif::{ProgressBar, ProgressDrawTarget};
 use std::fs;
-use std::io::{Error, ErrorKind};
+use std::io::{Error, ErrorKind, Read};
 use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::process::Command;
@@ -75,6 +75,40 @@ fn ping_and_wait(frontend_url: &str) -> DfxResult {
     })
 }
 
+// The frontend webserver is brought up by the bg process; thus, the fg process
+// needs to wait and verify it's up before exiting.
+// Because the user may have specified to start on port 0, here we wait for
+// webserver_port_path to get written to and modify the frontend_url so we
+// ping the correct address.
+fn fg_ping_and_wait(webserver_port_path: PathBuf, frontend_url: String) -> DfxResult {
+    let mut waiter = Delay::builder()
+        .timeout(std::time::Duration::from_secs(30))
+        .throttle(std::time::Duration::from_secs(1))
+        .build();
+    let mut runtime = Runtime::new().expect("Unable to create a runtime");
+    let port = runtime.block_on(async {
+        waiter.start();
+        let mut contents = String::new();
+        loop {
+            let tokio_file = tokio::fs::File::open(&webserver_port_path).await?;
+            let mut std_file = tokio_file.into_std().await;
+            std_file.read_to_string(&mut contents)?;
+            if !contents.is_empty() {
+                break;
+            }
+            waiter.wait()?;
+        }
+        Ok::<String, DfxError>(contents.clone())
+    })?;
+    let mut frontend_url_mod = frontend_url.clone();
+    let port_offset = frontend_url_mod
+        .as_str()
+        .rfind(':')
+        .ok_or_else(|| DfxError::MalformedFrontendUrl(frontend_url))?;
+    frontend_url_mod.replace_range((port_offset + 1).., port.as_str());
+    ping_and_wait(&frontend_url_mod)
+}
+
 // TODO(eftychis)/In progress: Rename to replica.
 /// Start the Internet Computer locally. Spawns a proxy to forward and
 /// manage browser requests. Responsible for running the network (one
@@ -89,7 +123,11 @@ pub fn exec(env: &dyn Environment, args: &ArgMatches<'_>) -> DfxResult {
     let (frontend_url, address_and_port) = frontend_address(args, &config)?;
     let webserver_port_path = temp_dir.join("webserver-port");
     std::fs::write(&webserver_port_path, "")?;
-    std::fs::write(&webserver_port_path, address_and_port.port().to_string())?;
+
+    // don't write to file since this arg means we send_background()
+    if !args.is_present("background") {
+        std::fs::write(&webserver_port_path, address_and_port.port().to_string())?;
+    }
 
     let client_pathbuf = env.get_cache().get_binary_command_path("replica")?;
     let ic_starter_pathbuf = env.get_cache().get_binary_command_path("ic-starter")?;
@@ -130,7 +168,7 @@ pub fn exec(env: &dyn Environment, args: &ArgMatches<'_>) -> DfxResult {
 
     if args.is_present("background") {
         send_background()?;
-        return Ok(());
+        return fg_ping_and_wait(webserver_port_path, frontend_url);
     }
 
     // Start the client.
diff --git a/src/dfx/src/lib/error/mod.rs b/src/dfx/src/lib/error/mod.rs
index 562dfa9b2a..145e26053f 100644
--- a/src/dfx/src/lib/error/mod.rs
+++ b/src/dfx/src/lib/error/mod.rs
@@ -144,7 +144,14 @@ pub enum DfxError {
     /// Could not save the contents of the file
     CouldNotSaveCanisterIds(String, std::io::Error),
 
+    /// Could not parse a string using Humanize
     HumanizeParseError(humanize_rs::ParseError),
+
+    /// An error occured when waiting using the Delay crate
+    WaiterError(delay::WaiterError),
+
+    /// The url for the frontend webserver is malformed
+    MalformedFrontendUrl(String),
 }
 
 /// The result of running a DFX command.
@@ -239,6 +246,9 @@ impl Display for DfxError {
             DfxError::CouldNotSaveCanisterIds(path, error) => {
                 f.write_fmt(format_args!("Failed to save {} due to: {}", path, error))?;
             }
+            DfxError::MalformedFrontendUrl(err) => {
+                f.write_fmt(format_args!("Malformed frontend url: {}", err))?;
+            }
             err => {
                 f.write_fmt(format_args!("An error occured:\n{:#?}", err))?;
             }
@@ -314,3 +324,9 @@ impl From<humanize_rs::ParseError> for DfxError {
         DfxError::HumanizeParseError(err)
     }
 }
+
+impl From<delay::WaiterError> for DfxError {
+    fn from(err: delay::WaiterError) -> DfxError {
+        DfxError::WaiterError(err)
+    }
+}

From 7f582438b7f66e01e59fd3bdf07b0ace570644c8 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Tue, 29 Sep 2020 12:25:41 -0700
Subject: [PATCH 17/19] remove sleep in dfx_start bats utility since we already
 wait in the source for webserver_port to be written

---
 e2e/bats/utils/_.bash | 1 -
 1 file changed, 1 deletion(-)

diff --git a/e2e/bats/utils/_.bash b/e2e/bats/utils/_.bash
index 1505a33083..97fca81011 100644
--- a/e2e/bats/utils/_.bash
+++ b/e2e/bats/utils/_.bash
@@ -65,7 +65,6 @@ dfx_start() {
         local dfx_config_root=.dfx/client-configuration
         printf "Configuration Root for DFX: %s\n" "${dfx_config_root}"
         test -f ${dfx_config_root}/client-1.port
-        sleep 5
         local port=$(cat ${dfx_config_root}/client-1.port)
 
         # Overwrite the default networks.local.bind 127.0.0.1:8000 with allocated port

From 7c38e6d0bb9a7b01328e1ccd6353500e3948898a Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Tue, 29 Sep 2020 15:37:17 -0700
Subject: [PATCH 18/19] update contributing adoc

---
 .github/CONTRIBUTING.adoc | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/CONTRIBUTING.adoc b/.github/CONTRIBUTING.adoc
index d65b06ba3f..0e927fb3cf 100644
--- a/.github/CONTRIBUTING.adoc
+++ b/.github/CONTRIBUTING.adoc
@@ -30,9 +30,7 @@ dfx 0.5.7-1-gad81116
 
 [source,bash]
 ----
-sdk $ nix-shell -A e2e-tests default.nix
-[nix-shell:~/d/sdk]$ cd e2e/bats
-[nix-shell:~/d/sdk]$ bats *.bash
+sdk $ nix-build -A e2e-tests default.nix
 ----
 
 ==== Running End-to-End Tests Locally Against Reference IC
@@ -42,9 +40,7 @@ https://github.com/dfinity-lab/ic-ref[reference implementation of the Internet C
 
 [source,bash]
 ----
-sdk $ nix-shell -A e2e-tests-ic-ref default.nix
-[nix-shell:~/d/sdk]$ cd e2e/bats
-[nix-shell:~/d/sdk]$ USE_IC_REF=1 bats *.bash
+sdk $ nix-build -A e2e-tests-ic-ref default.nix
 ----
 
 ==== Running `dfx` in a Debugger

From 6dfa4aa91105934c145414fcd9b8a1f7c4a17b64 Mon Sep 17 00:00:00 2001
From: Prithvi Shahi <prithvi@dfinity.org>
Date: Tue, 29 Sep 2020 16:54:40 -0700
Subject: [PATCH 19/19] update contributing adoc

---
 .github/CONTRIBUTING.adoc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/CONTRIBUTING.adoc b/.github/CONTRIBUTING.adoc
index 0e927fb3cf..f9d54e979a 100644
--- a/.github/CONTRIBUTING.adoc
+++ b/.github/CONTRIBUTING.adoc
@@ -30,7 +30,9 @@ dfx 0.5.7-1-gad81116
 
 [source,bash]
 ----
-sdk $ nix-build -A e2e-tests default.nix
+sdk $ nix-shell -A ${name_of_any_e2e_test} e2e/bats/default.nix
+[nix-shell:~/d/sdk]$ cd e2e/bats
+[nix-shell:~/d/sdk]$ bats *.bash
 ----
 
 ==== Running End-to-End Tests Locally Against Reference IC
@@ -40,7 +42,9 @@ https://github.com/dfinity-lab/ic-ref[reference implementation of the Internet C
 
 [source,bash]
 ----
-sdk $ nix-build -A e2e-tests-ic-ref default.nix
+sdk $ nix-shell -A ${name_of_any_e2e_test} --arg use_ic_ref true e2e/bats/default.nix
+[nix-shell:~/d/sdk]$ cd e2e/bats
+[nix-shell:~/d/sdk]$ bats *.bash
 ----
 
 ==== Running `dfx` in a Debugger