From d5f0bf8106f5c2f3ffa537221774f33283769518 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Fri, 27 May 2022 13:25:00 -0700 Subject: [PATCH] feat: bootstrap: look up replica/emulator port --- CHANGELOG.adoc | 7 +++ e2e/tests-dfx/bitcoin.bash | 13 ++--- e2e/tests-dfx/canister_http.bash | 10 ++-- e2e/utils/_.bash | 8 +-- src/dfx/src/actors/icx_proxy.rs | 16 +++--- src/dfx/src/actors/mod.rs | 4 +- src/dfx/src/commands/bootstrap.rs | 83 +++++++++++++++++++++++++++---- src/dfx/src/commands/replica.rs | 29 ++++++++--- src/dfx/src/commands/start.rs | 23 ++++++--- 9 files changed, 137 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index cff15a3bcc..455407978d 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -136,6 +136,13 @@ a dfx process that is starting. dfx ping now uses the anonymous identity, and no longer requires dfx.json to be present. +=== dfx bootstrap now looks up the port of the local replica + +`dfx replica` writes the port of the running replica to one of these locations: + - .dfx/replica-configuration/replica-1.port + - .dfx/ic-ref.port +`dfx bootstrap` will now use this port value, so it's no longer necessary to edit dfx.json after running `dfx replica`. + == Dependencies === Rust Agent diff --git a/e2e/tests-dfx/bitcoin.bash b/e2e/tests-dfx/bitcoin.bash index 5e9bf75d6a..3ea1214a93 100644 --- a/e2e/tests-dfx/bitcoin.bash +++ b/e2e/tests-dfx/bitcoin.bash @@ -100,16 +100,9 @@ set_default_bitcoin_enabled() { "until curl --fail --verbose -o /dev/null http://localhost:\$(cat .dfx/replica-configuration/replica-1.port)/api/v2/status; do echo \"waiting for replica to restart on port \$(cat .dfx/replica-configuration/replica-1.port)\"; sleep 1; done" \ || (echo "replica did not restart" && echo "last replica port was $(get_replica_port)" && ps aux && exit 1) - # unfortunately bootstrap will never know about the new replica port. This makes dfx bypass bootstrap: - overwrite_webserver_port "$(get_replica_port)" - - echo "dfx.json configured for new replica:" - cat dfx.json - - timeout 15s sh -c \ - 'until dfx ping; do echo waiting for replica to restart; sleep 1; done' \ - || (echo "replica did not restart" && ps aux && exit 1) - wait_until_replica_healthy + # bootstrap doesn't detect the new replica port, so we have to restart it + stop_dfx_bootstrap + dfx_bootstrap # Sometimes initially get an error like: # IC0304: Attempt to execute a message on canister <>> which contains no Wasm module diff --git a/e2e/tests-dfx/canister_http.bash b/e2e/tests-dfx/canister_http.bash index 385fd5a0e2..7d3bfd9746 100644 --- a/e2e/tests-dfx/canister_http.bash +++ b/e2e/tests-dfx/canister_http.bash @@ -90,13 +90,9 @@ set_default_canister_http_enabled() { "until curl --fail --verbose -o /dev/null http://localhost:\$(cat .dfx/replica-configuration/replica-1.port)/api/v2/status; do echo \"waiting for replica to restart on port \$(cat .dfx/replica-configuration/replica-1.port)\"; sleep 1; done" \ || (echo "replica did not restart" && echo "last replica port was $(get_replica_port)" && ps aux && exit 1) - # unfortunately bootstrap will never know about the new replica port. This makes dfx bypass bootstrap: - overwrite_webserver_port "$(get_replica_port)" - - timeout 15s sh -c \ - 'until dfx ping; do echo waiting for replica to restart; sleep 1; done' \ - || (echo "replica did not restart" && ps aux && exit 1) - wait_until_replica_healthy + # bootstrap doesn't detect the new replica port, so we have to restart it + stop_dfx_bootstrap + dfx_bootstrap # Sometimes initially get an error like: # IC0304: Attempt to execute a message on canister <>> which contains no Wasm module diff --git a/e2e/utils/_.bash b/e2e/utils/_.bash index a2441f9fe6..7d5e1f8af5 100644 --- a/e2e/utils/_.bash +++ b/e2e/utils/_.bash @@ -177,8 +177,6 @@ dfx_replica() { local replica_port=$(cat ${dfx_config_root}/replica-1.port) fi - # Overwrite the default networks.local.bind 127.0.0.1:8000 with allocated port - cat <<<$(jq .networks.local.bind=\"127.0.0.1:${replica_port}\" dfx.json) >dfx.json printf "Replica Configured Port: %s\n" "${replica_port}" @@ -186,7 +184,9 @@ dfx_replica() { "until nc -z localhost ${replica_port}; do echo waiting for replica; sleep 1; done" \ || (echo "could not connect to replica on port ${replica_port}" && exit 1) - wait_until_replica_healthy + # ping the replica directly, because the bootstrap (that launches icx-proxy, which dfx ping usually connects to) + # is not running yet + dfx ping --wait-healthy "http://127.0.0.1:${replica_port}" } dfx_bootstrap() { @@ -205,6 +205,8 @@ dfx_bootstrap() { 'until nc -z localhost $(cat .dfx/webserver-port); do echo waiting for webserver; sleep 1; done' \ || (echo "could not connect to webserver on port $(cat .dfx/webserver-port)" && exit 1) + wait_until_replica_healthy + local proxy_port=$(cat .dfx/proxy-port) printf "Proxy Configured Port: %s\n", "${proxy_port}" diff --git a/src/dfx/src/actors/icx_proxy.rs b/src/dfx/src/actors/icx_proxy.rs index 0b00bca5fc..580e82fa53 100644 --- a/src/dfx/src/actors/icx_proxy.rs +++ b/src/dfx/src/actors/icx_proxy.rs @@ -40,7 +40,7 @@ pub struct IcxProxyConfig { pub proxy_port: u16, /// fixed replica addresses - pub providers: Vec, + pub replica_urls: Vec, /// does the icx-proxy need to fetch the root key pub fetch_root_key: bool, @@ -80,7 +80,7 @@ impl IcxProxy { } } - fn start_icx_proxy(&mut self, providers: Vec) -> DfxResult { + fn start_icx_proxy(&mut self, replica_urls: Vec) -> DfxResult { let logger = self.logger.clone(); let config = &self.config.icx_proxy_config; let proxy_port = config.proxy_port; @@ -93,7 +93,7 @@ impl IcxProxy { icx_proxy_start_thread( logger, config.bind, - providers, + replica_urls, proxy_port, icx_proxy_path, icx_proxy_pid_path.clone(), @@ -135,8 +135,8 @@ impl Actor for IcxProxy { .shutdown_controller .do_send(ShutdownSubscribe(ctx.address().recipient::())); - if !self.config.icx_proxy_config.providers.is_empty() { - self.start_icx_proxy(self.config.icx_proxy_config.providers.clone()) + if !self.config.icx_proxy_config.replica_urls.is_empty() { + self.start_icx_proxy(self.config.icx_proxy_config.replica_urls.clone()) .expect("Could not start icx-proxy"); } } @@ -187,7 +187,7 @@ impl Handler for IcxProxy { fn icx_proxy_start_thread( logger: Logger, address: SocketAddr, - providers: Vec, + replica_urls: Vec, proxy_port: u16, icx_proxy_path: PathBuf, icx_proxy_pid_path: PathBuf, @@ -213,8 +213,8 @@ fn icx_proxy_start_thread( let address = format!("{}", &address); let proxy = format!("http://localhost:{}", proxy_port); cmd.args(&["--address", &address, "--proxy", &proxy]); - for provider in providers { - let s = format!("{}", provider); + for url in replica_urls { + let s = format!("{}", url); cmd.args(&["--replica", &s]); } cmd.stdout(std::process::Stdio::inherit()); diff --git a/src/dfx/src/actors/mod.rs b/src/dfx/src/actors/mod.rs index 6e31e84d5e..aa56d3d099 100644 --- a/src/dfx/src/actors/mod.rs +++ b/src/dfx/src/actors/mod.rs @@ -86,12 +86,10 @@ pub fn start_canister_http_adapter_actor( pub fn start_emulator_actor( env: &dyn Environment, shutdown_controller: Addr, + emulator_port_path: PathBuf, ) -> DfxResult> { let ic_ref_path = env.get_cache().get_binary_command_path("ic-ref")?; - let temp_dir = env.get_temp_dir(); - let emulator_port_path = temp_dir.join("ic-ref.port"); - // Touch the port file. This ensures it is empty prior to // handing it over to ic-ref. If we read the file and it has // contents we shall assume it is due to our spawned ic-ref diff --git a/src/dfx/src/commands/bootstrap.rs b/src/dfx/src/commands/bootstrap.rs index 81ff54c1fd..7480aa8e47 100644 --- a/src/dfx/src/commands/bootstrap.rs +++ b/src/dfx/src/commands/bootstrap.rs @@ -1,3 +1,5 @@ +use crate::actors::icx_proxy::IcxProxyConfig; +use crate::actors::{start_icx_proxy_actor, start_shutdown_controller}; use crate::config::dfinity::{ConfigDefaults, ConfigDefaultsBootstrap}; use crate::lib::environment::Environment; use crate::lib::error::DfxResult; @@ -6,13 +8,13 @@ use crate::lib::provider::{get_network_descriptor, LocalBindDetermination}; use crate::lib::webserver::run_webserver; use crate::util::get_reusable_socket_addr; -use crate::actors::icx_proxy::IcxProxyConfig; -use crate::actors::{start_icx_proxy_actor, start_shutdown_controller}; use anyhow::{anyhow, Context, Error}; use clap::Parser; use fn_error_context::context; +use slog::info; use std::default::Default; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::path::Path; use url::Url; /// Starts the bootstrap server. @@ -60,11 +62,7 @@ pub fn exec(env: &dyn Environment, opts: BootstrapOpts) -> DfxResult { let build_output_root = build_output_root.join("canisters"); let icx_proxy_pid_file_path = env.get_temp_dir().join("icx-proxy-pid"); - let providers = get_providers(&network_descriptor)?; - let providers: Vec = providers - .iter() - .map(|uri| Url::parse(uri).unwrap()) - .collect(); + let replica_urls = get_replica_urls(env, &network_descriptor)?; // Since the user may have provided port "0", we need to grab a dynamically // allocated port and construct a resuable SocketAddr which the actix @@ -87,7 +85,7 @@ pub fn exec(env: &dyn Environment, opts: BootstrapOpts) -> DfxResult { ) })?; - verify_unique_ports(&providers, &socket_addr)?; + verify_unique_ports(&replica_urls, &socket_addr)?; let system = actix::System::new(); let _proxy = system @@ -114,7 +112,7 @@ pub fn exec(env: &dyn Environment, opts: BootstrapOpts) -> DfxResult { let icx_proxy_config = IcxProxyConfig { bind: socket_addr, proxy_port: webserver_bind.port(), - providers, + replica_urls, fetch_root_key: !network_descriptor.is_ic, }; @@ -215,16 +213,79 @@ fn get_port(config: &ConfigDefaultsBootstrap, port: Option<&str>) -> DfxResult DfxResult> { + if network_descriptor.name == "local" { + if let Some(port) = get_running_replica_port(env)? { + let local_server_descriptor = network_descriptor.local_server_descriptor()?; + let mut socket_addr = local_server_descriptor.bind_address; + socket_addr.set_port(port); + let url = format!("http://{}", socket_addr); + let url = Url::parse(&url)?; + return Ok(vec![url]); + } + } + get_providers(network_descriptor) +} + +fn get_running_replica_port(env: &dyn Environment) -> DfxResult> { + let logger = env.get_logger(); + // dfx start and dfx replica both write these as empty, and then + // populate one with a port. + let emulator_port_path = env.get_temp_dir().join("ic-ref.port"); + let replica_port_path = env + .get_temp_dir() + .join("replica-configuration") + .join("replica-1.port"); + + match read_port_from(&replica_port_path)? { + Some(port) => { + info!(logger, "Found local replica running on port {}", port); + Ok(Some(port)) + } + None => match read_port_from(&emulator_port_path)? { + Some(port) => { + info!(logger, "Found local emulator running on port {}", port); + Ok(Some(port)) + } + None => Ok(None), + }, + } +} + /// Gets the list of compute provider API endpoints. #[context("Failed to get providers for network '{}'.", network_descriptor.name)] -fn get_providers(network_descriptor: &NetworkDescriptor) -> DfxResult> { +fn get_providers(network_descriptor: &NetworkDescriptor) -> DfxResult> { network_descriptor .providers .iter() - .map(|url| Ok(format!("{}/api", url))) + .map(|url| parse_url(url)) .collect() } +#[context("Failed to parse url '{}'.", url)] +fn parse_url(url: &str) -> DfxResult { + Ok(Url::parse(url)?) +} + +#[context("Failed to read port value from {}", path.to_string_lossy())] +fn read_port_from(path: &Path) -> DfxResult> { + if path.exists() { + let s = std::fs::read_to_string(&path)?; + let s = s.trim(); + if s.is_empty() { + Ok(None) + } else { + let port = s.parse::()?; + Ok(Some(port)) + } + } else { + Ok(None) + } +} /// Gets the maximum amount of time, in seconds, the bootstrap server will wait for upstream /// requests to complete. First checks if the timeout was specified on the command-line using /// --timeout, otherwise checks if the timeout was specified in the dfx configuration file, diff --git a/src/dfx/src/commands/replica.rs b/src/dfx/src/commands/replica.rs index 79a6419213..fc51716880 100644 --- a/src/dfx/src/commands/replica.rs +++ b/src/dfx/src/commands/replica.rs @@ -12,10 +12,13 @@ use crate::commands::start::{ configure_btc_adapter_if_enabled, configure_canister_http_adapter_if_enabled, empty_writable_path, }; +use anyhow::Context; use clap::Parser; use fn_error_context::context; use std::default::Default; +use std::fs; use std::net::SocketAddr; +use std::path::PathBuf; /// Starts a local Internet Computer replica. #[derive(Parser)] @@ -43,15 +46,15 @@ pub struct ReplicaOpts { /// Gets the configuration options for the Internet Computer replica. #[context("Failed to get replica config.")] -fn get_config(env: &dyn Environment, opts: ReplicaOpts) -> DfxResult { +fn get_config( + env: &dyn Environment, + opts: ReplicaOpts, + replica_port_path: PathBuf, +) -> DfxResult { let config = get_config_from_file(env); let port = get_port(&config, opts.port)?; let mut http_handler: HttpHandlerConfig = Default::default(); if port == 0 { - let replica_port_path = env - .get_temp_dir() - .join("replica-configuration") - .join("replica-1.port"); http_handler.write_port_to = Some(replica_port_path); } else { http_handler.port = Some(port); @@ -101,6 +104,18 @@ pub fn exec(env: &dyn Environment, opts: ReplicaOpts) -> DfxResult { empty_writable_path(temp_dir.join("ic-canister-http-config.json"))?; let canister_http_adapter_socket_holder_path = temp_dir.join("ic-canister-http-socket-path"); + // dfx bootstrap will read these port files to find out which port to use, + // so we need to make sure only one has a valid port in it. + let replica_config_dir = temp_dir.join("replica-configuration"); + fs::create_dir_all(&replica_config_dir).with_context(|| { + format!( + "Failed to create replica config directory {}.", + replica_config_dir.display() + ) + })?; + let replica_port_path = empty_writable_path(replica_config_dir.join("replica-1.port"))?; + let emulator_port_path = empty_writable_path(temp_dir.join("ic-ref.port"))?; + let config = env.get_config_or_anyhow()?; let btc_adapter_config = configure_btc_adapter_if_enabled( config.get_config(), @@ -126,7 +141,7 @@ pub fn exec(env: &dyn Environment, opts: ReplicaOpts) -> DfxResult { system.block_on(async move { let shutdown_controller = start_shutdown_controller(env)?; if opts.emulator { - start_emulator_actor(env, shutdown_controller)?; + start_emulator_actor(env, shutdown_controller, emulator_port_path)?; } else { let (btc_adapter_ready_subscribe, btc_adapter_socket_path) = if let Some(ref btc_adapter_config) = btc_adapter_config { @@ -159,7 +174,7 @@ pub fn exec(env: &dyn Environment, opts: ReplicaOpts) -> DfxResult { (None, None) }; - let mut replica_config = get_config(env, opts)?; + let mut replica_config = get_config(env, opts, replica_port_path)?; if btc_adapter_config.is_some() { replica_config = replica_config.with_btc_adapter_enabled(); if let Some(btc_adapter_socket) = btc_adapter_socket_path { diff --git a/src/dfx/src/commands/start.rs b/src/dfx/src/commands/start.rs index d507e0de3e..8f1f74ddbb 100644 --- a/src/dfx/src/commands/start.rs +++ b/src/dfx/src/commands/start.rs @@ -192,6 +192,19 @@ pub fn exec( let icx_proxy_pid_file_path = empty_writable_path(temp_dir.join("icx-proxy-pid"))?; let webserver_port_path = empty_writable_path(temp_dir.join("webserver-port"))?; + // dfx bootstrap will read these port files to find out which port to use, + // so we need to make sure only one has a valid port in it. + let replica_config_dir = temp_dir.join("replica-configuration"); + fs::create_dir_all(&replica_config_dir).with_context(|| { + format!( + "Failed to create replica config directory {}.", + replica_config_dir.display() + ) + })?; + + let replica_port_path = empty_writable_path(replica_config_dir.join("replica-1.port"))?; + let emulator_port_path = empty_writable_path(temp_dir.join("ic-ref.port"))?; + let (frontend_url, address_and_port) = frontend_address(host, local_server_descriptor, background)?; @@ -236,7 +249,8 @@ pub fn exec( let shutdown_controller = start_shutdown_controller(env)?; let port_ready_subscribe: Recipient = if emulator { - let emulator = start_emulator_actor(env, shutdown_controller.clone())?; + let emulator = + start_emulator_actor(env, shutdown_controller.clone(), emulator_port_path)?; emulator.recipient() } else { let (btc_adapter_ready_subscribe, btc_adapter_socket_path) = @@ -270,11 +284,6 @@ pub fn exec( (None, None) }; - let replica_port_path = env - .get_temp_dir() - .join("replica-configuration") - .join("replica-1.port"); - let subnet_type = config .get_config() .get_defaults() @@ -310,7 +319,7 @@ pub fn exec( let icx_proxy_config = IcxProxyConfig { bind: address_and_port, proxy_port: webserver_bind.port(), - providers: vec![], + replica_urls: vec![], // will be determined after replica starts fetch_root_key: !network_descriptor.is_ic, };