diff --git a/e2e/bats/assets/transitive_deps_canisters/dfx.json b/e2e/bats/assets/transitive_deps_canisters/dfx.json index eef29a38cd..3472a6f366 100644 --- a/e2e/bats/assets/transitive_deps_canisters/dfx.json +++ b/e2e/bats/assets/transitive_deps_canisters/dfx.json @@ -23,6 +23,21 @@ "dependencies": ["canister_d"], "main": "./e/main.mo", "type": "motoko" + }, + "canister_f": { + "dependencies": ["canister_g", "canister_h"], + "main": "./f/main.mo", + "type": "motoko" + }, + "canister_g": { + "dependencies": ["canister_a"], + "main": "./g/main.mo", + "type": "motoko" + }, + "canister_h": { + "dependencies": ["canister_a"], + "main": "./h/main.mo", + "type": "motoko" } }, "defaults": { diff --git a/e2e/bats/assets/transitive_deps_canisters/f/main.mo b/e2e/bats/assets/transitive_deps_canisters/f/main.mo new file mode 100644 index 0000000000..a0c21ef113 --- /dev/null +++ b/e2e/bats/assets/transitive_deps_canisters/f/main.mo @@ -0,0 +1,5 @@ +actor { + public func greet(name : Text) : async Text { + return "Bună, " # name # "!"; + }; +}; diff --git a/e2e/bats/assets/transitive_deps_canisters/g/main.mo b/e2e/bats/assets/transitive_deps_canisters/g/main.mo new file mode 100644 index 0000000000..4bc493f996 --- /dev/null +++ b/e2e/bats/assets/transitive_deps_canisters/g/main.mo @@ -0,0 +1,5 @@ +actor { + public func greet(name : Text) : async Text { + return "Ciao, " # name # "!"; + }; +}; diff --git a/e2e/bats/assets/transitive_deps_canisters/h/main.mo b/e2e/bats/assets/transitive_deps_canisters/h/main.mo new file mode 100644 index 0000000000..b2274da749 --- /dev/null +++ b/e2e/bats/assets/transitive_deps_canisters/h/main.mo @@ -0,0 +1,5 @@ +actor { + public func greet(name : Text) : async Text { + return "Olá, " # name # "!"; + }; +}; diff --git a/e2e/bats/build_granular.bash b/e2e/bats/build_granular.bash index c95f5cf641..ba6a408590 100644 --- a/e2e/bats/build_granular.bash +++ b/e2e/bats/build_granular.bash @@ -77,8 +77,15 @@ teardown() { install_asset transitive_deps_canisters dfx_start dfx canister create --all - assert_command dfx build canister_e - assert_match "Possible circular dependency detected during evaluation of canister_d's dependency on canister_e." + assert_command_fail dfx build canister_e + assert_match " There is a dependency cycle between canisters found at canister canister_e -> canister_d -> canister_e" +} + +@test "multiple non-cyclic dependency paths to the same canister are ok" { + install_asset transitive_deps_canisters + dfx_start + dfx canister create --all + assert_command dfx build canister_f } @test "the all flag builds everything" { diff --git a/e2e/bats/deploy.bash b/e2e/bats/deploy.bash new file mode 100644 index 0000000000..865a2d0273 --- /dev/null +++ b/e2e/bats/deploy.bash @@ -0,0 +1,58 @@ +#!/usr/bin/env bats + +load utils/_ + +setup() { + # We want to work from a temporary directory, different for every test. + cd $(mktemp -d -t dfx-e2e-XXXXXXXX) + export RUST_BACKTRACE=1 + + dfx_new hello + dfx_start +} + +teardown() { + dfx_stop +} + +@test "deploy from a fresh project" { + install_asset greet + assert_command dfx deploy + + assert_command dfx canister call hello greet '("Banzai")' + assert_eq '("Hello, Banzai!")' +} + +@test "deploy a canister without dependencies" { + install_asset greet + assert_command dfx deploy hello + assert_match 'Deploying: hello' + assert_not_match 'hello_assets' +} + +@test "deploy a canister with dependencies" { + install_asset greet + assert_command dfx deploy hello_assets + assert_match 'Deploying: hello hello_assets' +} + +@test "deploy a canister with non-circular shared dependencies" { + install_asset transitive_deps_canisters + assert_command dfx deploy canister_f + assert_match 'Deploying: canister_a canister_f canister_g canister_h' +} + +@test "report an error on attempt to deploy a canister with circular dependencies" { + install_asset transitive_deps_canisters + assert_command_fail dfx deploy canister_d + assert_match 'canister_d -> canister_e -> canister_d' +} + +@test "if already registered, try to upgrade then install" { + install_asset greet + assert_command dfx canister create --all + + assert_command dfx deploy + assert_match 'attempting install' +} + diff --git a/e2e/bats/utils/assertions.bash b/e2e/bats/utils/assertions.bash index df8773c847..12052a090f 100644 --- a/e2e/bats/utils/assertions.bash +++ b/e2e/bats/utils/assertions.bash @@ -70,6 +70,25 @@ assert_match() { | fail) } +# Asserts that a string does not contain another string, using regexp. +# Arguments: +# $1 - The regex to use to match. +# $2 - The string to match against (output). By default it will use +# $output. +assert_not_match() { + regex="$1" + if [[ $# < 2 ]]; then + text="$output" + else + text="$2" + fi + if [[ "$text" =~ $regex ]]; then + (batslib_print_kv_single_or_multi 10 "regex" "$regex" "actual" "$text" \ + | batslib_decorate "output matches but is expected not to" \ + | fail) + fi +} + # Asserts a command will timeout. This assertion will fail if the command finishes before # the timeout period. If the command fails, it will also fail. # Arguments: diff --git a/src/dfx/src/commands/build.rs b/src/dfx/src/commands/build.rs index a23b4eb482..46874001c3 100644 --- a/src/dfx/src/commands/build.rs +++ b/src/dfx/src/commands/build.rs @@ -56,9 +56,12 @@ pub fn exec(env: &dyn Environment, args: &ArgMatches<'_>) -> DfxResult { // Option can be None in which case --all was specified let some_canister = args.value_of("canister_name"); + let canister_names = config + .get_config() + .get_canister_names_with_dependencies(some_canister)?; // Get pool of canisters to build - let canister_pool = CanisterPool::load(&env, build_mode_check, some_canister)?; + let canister_pool = CanisterPool::load(&env, build_mode_check, &canister_names)?; // Create canisters on the replica and associate canister ids locally. if args.is_present("check") { diff --git a/src/dfx/src/commands/deploy.rs b/src/dfx/src/commands/deploy.rs new file mode 100644 index 0000000000..5cbccdfe1b --- /dev/null +++ b/src/dfx/src/commands/deploy.rs @@ -0,0 +1,31 @@ +use crate::lib::environment::Environment; +use crate::lib::error::DfxResult; +use crate::lib::message::UserMessage; +use crate::lib::operations::canister::deploy_canisters; +use crate::lib::provider::create_agent_environment; +use clap::{App, Arg, ArgMatches, SubCommand}; + +pub fn construct() -> App<'static, 'static> { + SubCommand::with_name("deploy") + .about(UserMessage::DeployCanister.to_str()) + .arg( + Arg::with_name("canister_name") + .takes_value(true) + .help(UserMessage::DeployCanisterName.to_str()) + .required(false), + ) + .arg( + Arg::with_name("network") + .help(UserMessage::CanisterComputeNetwork.to_str()) + .long("network") + .takes_value(true), + ) +} + +pub fn exec(env: &dyn Environment, args: &ArgMatches<'_>) -> DfxResult { + let env = create_agent_environment(env, args)?; + + let canister = args.value_of("canister_name"); + + deploy_canisters(&env, canister) +} diff --git a/src/dfx/src/commands/mod.rs b/src/dfx/src/commands/mod.rs index 5a6981b2e9..199bd0eac8 100644 --- a/src/dfx/src/commands/mod.rs +++ b/src/dfx/src/commands/mod.rs @@ -7,6 +7,7 @@ mod build; mod cache; mod canister; mod config; +mod deploy; mod identity; mod language_service; mod new; @@ -48,6 +49,7 @@ pub fn builtin() -> Vec { CliCommand::new(cache::construct(), cache::exec), CliCommand::new(canister::construct(), canister::exec), CliCommand::new(config::construct(), config::exec), + CliCommand::new(deploy::construct(), deploy::exec), CliCommand::new(identity::construct(), identity::exec), CliCommand::new(language_service::construct(), language_service::exec), CliCommand::new(new::construct(), new::exec), diff --git a/src/dfx/src/config/dfinity.rs b/src/dfx/src/config/dfinity.rs index 7a2dd79099..1c8c57daf7 100644 --- a/src/dfx/src/config/dfinity.rs +++ b/src/dfx/src/config/dfinity.rs @@ -1,9 +1,9 @@ #![allow(dead_code)] -use crate::lib::error::{DfxError, DfxResult}; +use crate::lib::error::{BuildErrorKind, DfxError, DfxResult}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::default::Default; use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; use std::path::{Path, PathBuf}; @@ -238,6 +238,72 @@ impl ConfigInterface { pub fn get_dfx(&self) -> Option { self.dfx.to_owned() } + + /// Return the names of the specified canister and all of its dependencies. + /// If none specified, return the names of all canisters. + pub fn get_canister_names_with_dependencies( + &self, + some_canister: Option<&str>, + ) -> DfxResult> { + let canister_map = (&self.canisters).as_ref().ok_or_else(|| { + DfxError::InvalidConfiguration("No canisters in the configuration file.".to_string()) + })?; + + let canister_names = match some_canister { + Some(specific_canister) => { + let mut names = HashSet::new(); + let mut path = vec![]; + add_dependencies(canister_map, &mut names, &mut path, specific_canister)?; + names.into_iter().collect() + } + None => canister_map.keys().cloned().collect(), + }; + + Ok(canister_names) + } +} + +fn add_dependencies( + all_canisters: &BTreeMap, + names: &mut HashSet, + path: &mut Vec, + canister_name: &str, +) -> DfxResult { + let inserted = names.insert(String::from(canister_name)); + + if !inserted { + return if path.contains(&String::from(canister_name)) { + path.push(String::from(canister_name)); + Err(DfxError::BuildError(BuildErrorKind::CircularDependency( + path.join(" -> "), + ))) + } else { + Ok(()) + }; + } + + let canister_config = all_canisters + .get(canister_name) + .ok_or_else(|| DfxError::CannotFindCanisterName(canister_name.to_string()))?; + + let deps = match canister_config.extras.get("dependencies") { + None => vec![], + Some(v) => Vec::::deserialize(v).map_err(|_| { + DfxError::InvalidConfiguration(String::from( + "Field 'dependencies' is of the wrong type", + )) + })?, + }; + + path.push(String::from(canister_name)); + + for canister in deps { + add_dependencies(all_canisters, names, path, &canister)?; + } + + path.pop(); + + Ok(()) } #[derive(Clone)] diff --git a/src/dfx/src/lib/error/mod.rs b/src/dfx/src/lib/error/mod.rs index 35a3f9d30a..a4b8293af7 100644 --- a/src/dfx/src/lib/error/mod.rs +++ b/src/dfx/src/lib/error/mod.rs @@ -69,7 +69,6 @@ pub enum DfxError { /// Argument provided is invalid. InvalidArgument(String), - #[allow(dead_code)] /// Configuration provided is invalid. InvalidConfiguration(String), /// Method called invalid. @@ -198,6 +197,9 @@ impl Display for DfxError { DfxError::InvalidArgument(e) => { f.write_fmt(format_args!("Invalid argument: {}", e))?; } + DfxError::InvalidConfiguration(e) => { + f.write_fmt(format_args!("Invalid configuration: {}", e))?; + } DfxError::InvalidData(e) => { f.write_fmt(format_args!("Invalid data: {}", e))?; } diff --git a/src/dfx/src/lib/message.rs b/src/dfx/src/lib/message.rs index fb096657a2..e35e9ff128 100644 --- a/src/dfx/src/lib/message.rs +++ b/src/dfx/src/lib/message.rs @@ -98,7 +98,11 @@ user_message!( ConfigureOptions => "Configures project options for your currently-selected project.", OptionName => "Specifies the name of the configuration option to set or read. Use the period delineated path to specify the option to set or read. If this is not mentioned, outputs the whole configuration.", OptionValue => "Specifies the new value to set. If you don't specify a value, the command displays the current value of the option from the configuration file.", - OptionFormat => "Specifies the format of the output. By default, it uses JSON.", + OptionFormat => "Specifies the format of the output. By default, the output format is JSON.", + + // dfx deploy + DeployCanister => "Deploys all or a specific canister from the code in your project. By default, all canisters are deployed.", + DeployCanisterName => "Specifies the name of the canister you want to deploy. If you don’t specify a canister name, all canisters defined in the dfx.json file are deployed.", // dfx identity mod ManageIdentity => "Manages identities used to communicate with the Internet Computer network. Setting an identity enables you to test user-based access controls.", diff --git a/src/dfx/src/lib/models/canister.rs b/src/dfx/src/lib/models/canister.rs index 0fbcfade8e..18ad2b995d 100644 --- a/src/dfx/src/lib/models/canister.rs +++ b/src/dfx/src/lib/models/canister.rs @@ -9,8 +9,7 @@ use crate::lib::models::canister_id_store::CanisterIdStore; use crate::util::{assets, check_candid_file}; use ic_types::principal::Principal as CanisterId; use petgraph::graph::{DiGraph, NodeIndex}; -use rand::{thread_rng, Rng, RngCore}; -use serde::Deserialize; +use rand::{thread_rng, RngCore}; use slog::Logger; use std::cell::RefCell; use std::collections::BTreeMap; @@ -96,8 +95,6 @@ struct PoolConstructHelper<'a> { canister_id_store: CanisterIdStore, generate_cid: bool, canisters_map: &'a mut Vec>, - visited_map: BTreeMap, - logger: &'a Logger, } impl CanisterPool { @@ -113,9 +110,6 @@ impl CanisterPool { pool_helper .canisters_map .insert(0, Arc::new(Canister::new(info, builder))); - pool_helper - .visited_map - .insert(canister_name.to_owned(), thread_rng().gen::()); Ok(()) } else { Err(DfxError::CouldNotFindBuilderForCanister( @@ -124,42 +118,10 @@ impl CanisterPool { } } - fn insert_with_dependencies( - canister_name: &str, - pool_helper: &mut PoolConstructHelper<'_>, - ) -> DfxResult<()> { - //insert this canister - CanisterPool::insert(canister_name, pool_helper)?; - - // recursively fetch direct and transitive dependencies - let canister_id = pool_helper.canister_id_store.get(canister_name)?; - let info = CanisterInfo::load(pool_helper.config, canister_name, Some(canister_id))?; - let deps = match info.get_extra_value("dependencies") { - None => vec![], - Some(v) => Vec::::deserialize(v).map_err(|_| { - DfxError::Unknown(String::from("Field 'dependencies' is of the wrong type")) - })?, - }; - - for canister in deps.iter() { - if !pool_helper.visited_map.contains_key(&canister.to_owned()) { - CanisterPool::insert_with_dependencies(canister, pool_helper)?; - } else { - slog::warn!( - pool_helper.logger, - "Possible circular dependency detected during evaluation of {}'s dependency on {}.", - &canister_name.to_owned(), - &canister.to_owned(), - ); - } - } - Ok(()) - } - pub fn load( env: &dyn Environment, generate_cid: bool, - some_canister: Option<&str>, + canister_names: &[String], ) -> DfxResult { let logger = env.get_logger().new(slog::o!()); let config = env @@ -174,20 +136,10 @@ impl CanisterPool { canister_id_store: CanisterIdStore::for_env(env)?, generate_cid, canisters_map: &mut canisters_map, - visited_map: BTreeMap::new(), - logger: env.get_logger(), }; - if let Some(canister_name) = some_canister { - CanisterPool::insert_with_dependencies(canister_name, &mut pool_helper)?; - } else { - // insert all canisters configured in dfx.json - let canisters = config.get_config().canisters.as_ref().ok_or_else(|| { - DfxError::Unknown("No canisters in the configuration file.".to_string()) - })?; - for (key, _value) in canisters.iter() { - CanisterPool::insert(key, &mut pool_helper)?; - } + for canister_name in canister_names { + CanisterPool::insert(canister_name, &mut pool_helper)?; } Ok(CanisterPool { diff --git a/src/dfx/src/lib/operations/canister/create_canister.rs b/src/dfx/src/lib/operations/canister/create_canister.rs index 2d2f47f82b..1f7c4904ff 100644 --- a/src/dfx/src/lib/operations/canister/create_canister.rs +++ b/src/dfx/src/lib/operations/canister/create_canister.rs @@ -1,17 +1,17 @@ use crate::lib::environment::Environment; use crate::lib::error::{DfxError, DfxResult}; use crate::lib::models::canister_id_store::CanisterIdStore; -use crate::lib::progress_bar::ProgressBar; use crate::lib::provider::get_network_context; use crate::lib::waiter::create_waiter; use ic_agent::ManagementCanister; +use slog::info; use std::format; use tokio::runtime::Runtime; pub fn create_canister(env: &dyn Environment, canister_name: &str) -> DfxResult { - let message = format!("Creating canister {:?}...", canister_name); - let b = ProgressBar::new_spinner(&message); + let log = env.get_logger(); + info!(log, "Creating canister {:?}...", canister_name); env.get_config() .ok_or(DfxError::CommandMustBeRunInAProject)?; @@ -34,23 +34,25 @@ pub fn create_canister(env: &dyn Environment, canister_name: &str) -> DfxResult match canister_id_store.find(&canister_name) { Some(canister_id) => { - let message = format!( + info!( + log, "{:?} canister was already created {}and has canister id: {:?}", canister_name, non_default_network, canister_id.to_text() ); - b.finish_with_message(&message); Ok(()) } None => { let cid = runtime.block_on(mgr.create_canister(create_waiter()))?; let canister_id = cid.to_text(); - let message = format!( + info!( + log, "{:?} canister created {}with canister id: {:?}", - canister_name, non_default_network, canister_id + canister_name, + non_default_network, + canister_id ); - b.finish_with_message(&message); canister_id_store.add(&canister_name, canister_id) } }?; diff --git a/src/dfx/src/lib/operations/canister/deploy_canisters.rs b/src/dfx/src/lib/operations/canister/deploy_canisters.rs new file mode 100644 index 0000000000..71c1b99220 --- /dev/null +++ b/src/dfx/src/lib/operations/canister/deploy_canisters.rs @@ -0,0 +1,137 @@ +use crate::config::dfinity::Config; +use crate::lib::builders::BuildConfig; +use crate::lib::canister_info::CanisterInfo; +use crate::lib::environment::Environment; +use crate::lib::error::{DfxError, DfxResult}; +use crate::lib::models::canister::CanisterPool; +use crate::lib::models::canister_id_store::CanisterIdStore; +use crate::lib::operations::canister::create_canister; +use crate::lib::operations::canister::install_canister; +use ic_agent::{AgentError, InstallMode}; +use slog::{info, warn}; +use tokio::runtime::Runtime; + +pub fn deploy_canisters(env: &dyn Environment, some_canister: Option<&str>) -> DfxResult { + let log = env.get_logger(); + + let config = env + .get_config() + .ok_or(DfxError::CommandMustBeRunInAProject)?; + let initial_canister_id_store = CanisterIdStore::for_env(env)?; + + let canister_names = canisters_to_deploy(&config, some_canister)?; + if some_canister.is_some() { + info!(log, "Deploying: {}", canister_names.join(" ")); + } else { + info!(log, "Deploying all canisters."); + } + + register_canisters(env, &canister_names, &initial_canister_id_store)?; + + build_canisters(env, &canister_names, &config)?; + + install_canisters(env, &canister_names, &initial_canister_id_store, &config)?; + + info!(log, "Deployed canisters."); + + Ok(()) +} + +fn canisters_to_deploy(config: &Config, some_canister: Option<&str>) -> DfxResult> { + let mut canister_names = config + .get_config() + .get_canister_names_with_dependencies(some_canister)?; + canister_names.sort(); + Ok(canister_names) +} + +fn register_canisters( + env: &dyn Environment, + canister_names: &[String], + canister_id_store: &CanisterIdStore, +) -> DfxResult { + let canisters_to_create = canister_names + .iter() + .filter(|n| canister_id_store.find(&n).is_none()) + .cloned() + .collect::>(); + if canisters_to_create.is_empty() { + info!(env.get_logger(), "All canisters have already been created."); + } else { + info!(env.get_logger(), "Creating canisters..."); + for canister_name in &canisters_to_create { + create_canister(env, &canister_name)?; + } + } + Ok(()) +} + +fn build_canisters(env: &dyn Environment, canister_names: &[String], config: &Config) -> DfxResult { + info!(env.get_logger(), "Building canisters..."); + let build_mode_check = false; + let canister_pool = CanisterPool::load(env, build_mode_check, &canister_names)?; + + canister_pool.build_or_fail(BuildConfig::from_config(&config)?) +} + +fn install_canisters( + env: &dyn Environment, + canister_names: &[String], + initial_canister_id_store: &CanisterIdStore, + config: &Config, +) -> DfxResult { + info!(env.get_logger(), "Installing canisters..."); + + let agent = env + .get_agent() + .ok_or(DfxError::CommandMustBeRunInAProject)?; + + let mut runtime = Runtime::new().expect("Unable to create a runtime"); + + let canister_id_store = CanisterIdStore::for_env(env)?; + + for canister_name in canister_names { + let (first_mode, second_mode) = match initial_canister_id_store.find(&canister_name) { + Some(_) => (InstallMode::Upgrade, InstallMode::Install), + None => (InstallMode::Install, InstallMode::Upgrade), + }; + + let canister_id = canister_id_store.get(&canister_name)?; + let canister_info = CanisterInfo::load(&config, &canister_name, Some(canister_id))?; + let compute_allocation = None; + let result = runtime.block_on(install_canister( + env, + &agent, + &canister_info, + compute_allocation, + first_mode, + )); + match result { + Err(DfxError::AgentError(AgentError::ReplicaError { + reject_code, + reject_message: _, + })) if reject_code == 3 || reject_code == 5 => { + // 3: tried to upgrade a canister that has not been created + // 5: tried to install a canister that was already installed + let mode_description = match second_mode { + InstallMode::Install => "install", + _ => "upgrade", + }; + warn!( + env.get_logger(), + "replica error. attempting {}", mode_description + ); + runtime.block_on(install_canister( + env, + &agent, + &canister_info, + compute_allocation, + second_mode, + )) + } + other => other, + }?; + } + + Ok(()) +} diff --git a/src/dfx/src/lib/operations/canister/mod.rs b/src/dfx/src/lib/operations/canister/mod.rs index 0c8c5d56c1..e584e09e3b 100644 --- a/src/dfx/src/lib/operations/canister/mod.rs +++ b/src/dfx/src/lib/operations/canister/mod.rs @@ -1,5 +1,7 @@ mod create_canister; +mod deploy_canisters; mod install_canister; pub use create_canister::create_canister; +pub use deploy_canisters::deploy_canisters; pub use install_canister::install_canister;