feat: bootstrap: look up replica/emulator port

CHANGELOG.adoc

@@ -136,6 +136,13 @@ a dfx process that is starting.
 
 dfx ping now uses the anonymous identity, and no longer requires dfx.json to be present.
 
+=== dfx bootstrap now looks up the port of the local replica
+
+`dfx replica` writes the port of the running replica to one of these locations:
+  - .dfx/replica-configuration/replica-1.port
+  - .dfx/ic-ref.port
+`dfx bootstrap` will now use this port value, so it's no longer necessary to edit dfx.json after running `dfx replica`.
+
 == Dependencies
 
 === Rust Agent

e2e/tests-dfx/bitcoin.bash

@@ -100,16 +100,9 @@ set_default_bitcoin_enabled() {
       "until curl --fail --verbose -o /dev/null http://localhost:\$(cat .dfx/replica-configuration/replica-1.port)/api/v2/status; do echo \"waiting for replica to restart on port \$(cat .dfx/replica-configuration/replica-1.port)\"; sleep 1; done" \
       || (echo "replica did not restart" && echo "last replica port was $(get_replica_port)" && ps aux && exit 1)
 
-    # unfortunately bootstrap will never know about the new replica port.  This makes dfx bypass bootstrap:
-    overwrite_webserver_port "$(get_replica_port)"
-
-    echo "dfx.json configured for new replica:"
-    cat dfx.json
-
-    timeout 15s sh -c \
-      'until dfx ping; do echo waiting for replica to restart; sleep 1; done' \
-      || (echo "replica did not restart" && ps aux && exit 1)
-    wait_until_replica_healthy
+    # bootstrap doesn't detect the new replica port, so we have to restart it
+    stop_dfx_bootstrap
+    dfx_bootstrap
 
     # Sometimes initially get an error like:
     #     IC0304: Attempt to execute a message on canister <>> which contains no Wasm module

e2e/tests-dfx/canister_http.bash

@@ -90,13 +90,9 @@ set_default_canister_http_enabled() {
       "until curl --fail --verbose -o /dev/null http://localhost:\$(cat .dfx/replica-configuration/replica-1.port)/api/v2/status; do echo \"waiting for replica to restart on port \$(cat .dfx/replica-configuration/replica-1.port)\"; sleep 1; done" \
       || (echo "replica did not restart" && echo "last replica port was $(get_replica_port)" && ps aux && exit 1)
 
-    # unfortunately bootstrap will never know about the new replica port. This makes dfx bypass bootstrap:
-    overwrite_webserver_port "$(get_replica_port)"
-
-    timeout 15s sh -c \
-      'until dfx ping; do echo waiting for replica to restart; sleep 1; done' \
-      || (echo "replica did not restart" && ps aux && exit 1)
-    wait_until_replica_healthy
+    # bootstrap doesn't detect the new replica port, so we have to restart it
+    stop_dfx_bootstrap
+    dfx_bootstrap
 
     # Sometimes initially get an error like:
     #     IC0304: Attempt to execute a message on canister <>> which contains no Wasm module

e2e/utils/_.bash

@@ -177,16 +177,16 @@ dfx_replica() {
         local replica_port=$(cat ${dfx_config_root}/replica-1.port)
 
     fi
-    # Overwrite the default networks.local.bind 127.0.0.1:8000 with allocated port
-    cat <<<$(jq .networks.local.bind=\"127.0.0.1:${replica_port}\" dfx.json) >dfx.json
 
     printf "Replica Configured Port: %s\n" "${replica_port}"
 
     timeout 5 sh -c \
         "until nc -z localhost ${replica_port}; do echo waiting for replica; sleep 1; done" \
         || (echo "could not connect to replica on port ${replica_port}" && exit 1)
 
-    wait_until_replica_healthy
+    # ping the replica directly, because the bootstrap (that launches icx-proxy, which dfx ping usually connects to)
+    # is not running yet
+    dfx ping --wait-healthy "http://127.0.0.1:${replica_port}"
 }
 
 dfx_bootstrap() {
@@ -205,6 +205,8 @@ dfx_bootstrap() {
         'until nc -z localhost $(cat .dfx/webserver-port); do echo waiting for webserver; sleep 1; done' \
         || (echo "could not connect to webserver on port $(cat .dfx/webserver-port)" && exit 1)
 
+    wait_until_replica_healthy
+
     local proxy_port=$(cat .dfx/proxy-port)
     printf "Proxy Configured Port: %s\n", "${proxy_port}"
 

src/dfx/src/actors/icx_proxy.rs

@@ -40,7 +40,7 @@ pub struct IcxProxyConfig {
     pub proxy_port: u16,
 
     /// fixed replica addresses
-    pub providers: Vec<Url>,
+    pub replica_urls: Vec<Url>,
 
     /// does the icx-proxy need to fetch the root key
     pub fetch_root_key: bool,
@@ -80,7 +80,7 @@ impl IcxProxy {
         }
     }
 
-    fn start_icx_proxy(&mut self, providers: Vec<Url>) -> DfxResult {
+    fn start_icx_proxy(&mut self, replica_urls: Vec<Url>) -> DfxResult {
         let logger = self.logger.clone();
         let config = &self.config.icx_proxy_config;
         let proxy_port = config.proxy_port;
@@ -93,7 +93,7 @@ impl IcxProxy {
             icx_proxy_start_thread(
                 logger,
                 config.bind,
-                providers,
+                replica_urls,
                 proxy_port,
                 icx_proxy_path,
                 icx_proxy_pid_path.clone(),
@@ -135,8 +135,8 @@ impl Actor for IcxProxy {
             .shutdown_controller
             .do_send(ShutdownSubscribe(ctx.address().recipient::<Shutdown>()));
 
-        if !self.config.icx_proxy_config.providers.is_empty() {
-            self.start_icx_proxy(self.config.icx_proxy_config.providers.clone())
+        if !self.config.icx_proxy_config.replica_urls.is_empty() {
+            self.start_icx_proxy(self.config.icx_proxy_config.replica_urls.clone())
                 .expect("Could not start icx-proxy");
         }
     }
@@ -187,7 +187,7 @@ impl Handler<Shutdown> for IcxProxy {
 fn icx_proxy_start_thread(
     logger: Logger,
     address: SocketAddr,
-    providers: Vec<Url>,
+    replica_urls: Vec<Url>,
     proxy_port: u16,
     icx_proxy_path: PathBuf,
     icx_proxy_pid_path: PathBuf,
@@ -213,8 +213,8 @@ fn icx_proxy_start_thread(
         let address = format!("{}", &address);
         let proxy = format!("http://localhost:{}", proxy_port);
         cmd.args(&["--address", &address, "--proxy", &proxy]);
-        for provider in providers {
-            let s = format!("{}", provider);
+        for url in replica_urls {
+            let s = format!("{}", url);
             cmd.args(&["--replica", &s]);
         }
         cmd.stdout(std::process::Stdio::inherit());

src/dfx/src/actors/mod.rs

@@ -86,12 +86,10 @@ pub fn start_canister_http_adapter_actor(
 pub fn start_emulator_actor(
     env: &dyn Environment,
     shutdown_controller: Addr<ShutdownController>,
+    emulator_port_path: PathBuf,
 ) -> DfxResult<Addr<Emulator>> {
     let ic_ref_path = env.get_cache().get_binary_command_path("ic-ref")?;
 
-    let temp_dir = env.get_temp_dir();
-    let emulator_port_path = temp_dir.join("ic-ref.port");
-
     // Touch the port file. This ensures it is empty prior to
     // handing it over to ic-ref. If we read the file and it has
     // contents we shall assume it is due to our spawned ic-ref

src/dfx/src/commands/bootstrap.rs

@@ -1,3 +1,5 @@
+use crate::actors::icx_proxy::IcxProxyConfig;
+use crate::actors::{start_icx_proxy_actor, start_shutdown_controller};
 use crate::config::dfinity::{ConfigDefaults, ConfigDefaultsBootstrap};
 use crate::lib::environment::Environment;
 use crate::lib::error::DfxResult;
@@ -6,13 +8,13 @@ use crate::lib::provider::{get_network_descriptor, LocalBindDetermination};
 use crate::lib::webserver::run_webserver;
 use crate::util::get_reusable_socket_addr;
 
-use crate::actors::icx_proxy::IcxProxyConfig;
-use crate::actors::{start_icx_proxy_actor, start_shutdown_controller};
 use anyhow::{anyhow, Context, Error};
 use clap::Parser;
 use fn_error_context::context;
+use slog::info;
 use std::default::Default;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::path::Path;
 use url::Url;
 
 /// Starts the bootstrap server.
@@ -60,11 +62,7 @@ pub fn exec(env: &dyn Environment, opts: BootstrapOpts) -> DfxResult {
     let build_output_root = build_output_root.join("canisters");
     let icx_proxy_pid_file_path = env.get_temp_dir().join("icx-proxy-pid");
 
-    let providers = get_providers(&network_descriptor)?;
-    let providers: Vec<Url> = providers
-        .iter()
-        .map(|uri| Url::parse(uri).unwrap())
-        .collect();
+    let replica_urls = get_replica_urls(env, &network_descriptor)?;
 
     // Since the user may have provided port "0", we need to grab a dynamically
     // allocated port and construct a resuable SocketAddr which the actix
@@ -87,7 +85,7 @@ pub fn exec(env: &dyn Environment, opts: BootstrapOpts) -> DfxResult {
         )
     })?;
 
-    verify_unique_ports(&providers, &socket_addr)?;
+    verify_unique_ports(&replica_urls, &socket_addr)?;
 
     let system = actix::System::new();
     let _proxy = system
@@ -114,7 +112,7 @@ pub fn exec(env: &dyn Environment, opts: BootstrapOpts) -> DfxResult {
             let icx_proxy_config = IcxProxyConfig {
                 bind: socket_addr,
                 proxy_port: webserver_bind.port(),
-                providers,
+                replica_urls,
                 fetch_root_key: !network_descriptor.is_ic,
             };
 
@@ -215,16 +213,79 @@ fn get_port(config: &ConfigDefaultsBootstrap, port: Option<&str>) -> DfxResult<u
         .context("Invalid argument: Invalid port number.")
 }
 
+#[context("Failed to determine replica urls.")]
+fn get_replica_urls(
+    env: &dyn Environment,
+    network_descriptor: &NetworkDescriptor,
+) -> DfxResult<Vec<Url>> {
+    if network_descriptor.name == "local" {
+        if let Some(port) = get_running_replica_port(env)? {
+            let local_server_descriptor = network_descriptor.local_server_descriptor()?;
+            let mut socket_addr = local_server_descriptor.bind_address;
+            socket_addr.set_port(port);
+            let url = format!("http://{}", socket_addr);
+            let url = Url::parse(&url)?;
+            return Ok(vec![url]);
+        }
+    }
+    get_providers(network_descriptor)
+}
+
+fn get_running_replica_port(env: &dyn Environment) -> DfxResult<Option<u16>> {
+    let logger = env.get_logger();
+    // dfx start and dfx replica both write these as empty, and then
+    // populate one with a port.
+    let emulator_port_path = env.get_temp_dir().join("ic-ref.port");
+    let replica_port_path = env
+        .get_temp_dir()
+        .join("replica-configuration")
+        .join("replica-1.port");
+
+    match read_port_from(&replica_port_path)? {
+        Some(port) => {
+            info!(logger, "Found local replica running on port {}", port);
+            Ok(Some(port))
+        }
+        None => match read_port_from(&emulator_port_path)? {
+            Some(port) => {
+                info!(logger, "Found local emulator running on port {}", port);
+                Ok(Some(port))
+            }
+            None => Ok(None),
+        },
+    }
+}
+
 /// Gets the list of compute provider API endpoints.
 #[context("Failed to get providers for network '{}'.", network_descriptor.name)]
-fn get_providers(network_descriptor: &NetworkDescriptor) -> DfxResult<Vec<String>> {
+fn get_providers(network_descriptor: &NetworkDescriptor) -> DfxResult<Vec<Url>> {
     network_descriptor
         .providers
         .iter()
-        .map(|url| Ok(format!("{}/api", url)))
+        .map(|url| parse_url(url))
         .collect()
 }
 
+#[context("Failed to parse url '{}'.", url)]
+fn parse_url(url: &str) -> DfxResult<Url> {
+    Ok(Url::parse(url)?)
+}
+
+#[context("Failed to read port value from {}", path.to_string_lossy())]
+fn read_port_from(path: &Path) -> DfxResult<Option<u16>> {
+    if path.exists() {
+        let s = std::fs::read_to_string(&path)?;
+        let s = s.trim();
+        if s.is_empty() {
+            Ok(None)
+        } else {
+            let port = s.parse::<u16>()?;
+            Ok(Some(port))
+        }
+    } else {
+        Ok(None)
+    }
+}
 /// Gets the maximum amount of time, in seconds, the bootstrap server will wait for upstream
 /// requests to complete. First checks if the timeout was specified on the command-line using
 /// --timeout, otherwise checks if the timeout was specified in the dfx configuration file,

src/dfx/src/commands/replica.rs

@@ -12,10 +12,13 @@ use crate::commands::start::{
     configure_btc_adapter_if_enabled, configure_canister_http_adapter_if_enabled,
     empty_writable_path,
 };
+use anyhow::Context;
 use clap::Parser;
 use fn_error_context::context;
 use std::default::Default;
+use std::fs;
 use std::net::SocketAddr;
+use std::path::PathBuf;
 
 /// Starts a local Internet Computer replica.
 #[derive(Parser)]
@@ -43,15 +46,15 @@ pub struct ReplicaOpts {
 
 /// Gets the configuration options for the Internet Computer replica.
 #[context("Failed to get replica config.")]
-fn get_config(env: &dyn Environment, opts: ReplicaOpts) -> DfxResult<ReplicaConfig> {
+fn get_config(
+    env: &dyn Environment,
+    opts: ReplicaOpts,
+    replica_port_path: PathBuf,
+) -> DfxResult<ReplicaConfig> {
     let config = get_config_from_file(env);
     let port = get_port(&config, opts.port)?;
     let mut http_handler: HttpHandlerConfig = Default::default();
     if port == 0 {
-        let replica_port_path = env
-            .get_temp_dir()
-            .join("replica-configuration")
-            .join("replica-1.port");
         http_handler.write_port_to = Some(replica_port_path);
     } else {
         http_handler.port = Some(port);
@@ -101,6 +104,18 @@ pub fn exec(env: &dyn Environment, opts: ReplicaOpts) -> DfxResult {
         empty_writable_path(temp_dir.join("ic-canister-http-config.json"))?;
     let canister_http_adapter_socket_holder_path = temp_dir.join("ic-canister-http-socket-path");
 
+    // dfx bootstrap will read these port files to find out which port to use,
+    // so we need to make sure only one has a valid port in it.
+    let replica_config_dir = temp_dir.join("replica-configuration");
+    fs::create_dir_all(&replica_config_dir).with_context(|| {
+        format!(
+            "Failed to create replica config directory {}.",
+            replica_config_dir.display()
+        )
+    })?;
+    let replica_port_path = empty_writable_path(replica_config_dir.join("replica-1.port"))?;
+    let emulator_port_path = empty_writable_path(temp_dir.join("ic-ref.port"))?;
+
     let config = env.get_config_or_anyhow()?;
     let btc_adapter_config = configure_btc_adapter_if_enabled(
         config.get_config(),
@@ -126,7 +141,7 @@ pub fn exec(env: &dyn Environment, opts: ReplicaOpts) -> DfxResult {
     system.block_on(async move {
         let shutdown_controller = start_shutdown_controller(env)?;
         if opts.emulator {
-            start_emulator_actor(env, shutdown_controller)?;
+            start_emulator_actor(env, shutdown_controller, emulator_port_path)?;
         } else {
             let (btc_adapter_ready_subscribe, btc_adapter_socket_path) =
                 if let Some(ref btc_adapter_config) = btc_adapter_config {
@@ -159,7 +174,7 @@ pub fn exec(env: &dyn Environment, opts: ReplicaOpts) -> DfxResult {
                     (None, None)
                 };
 
-            let mut replica_config = get_config(env, opts)?;
+            let mut replica_config = get_config(env, opts, replica_port_path)?;
             if btc_adapter_config.is_some() {
                 replica_config = replica_config.with_btc_adapter_enabled();
                 if let Some(btc_adapter_socket) = btc_adapter_socket_path {