diff --git a/cli/src/cmd/forge/coverage.rs b/cli/src/cmd/forge/coverage.rs index d7c89ea88a2c5..ebcbd3e68f049 100644 --- a/cli/src/cmd/forge/coverage.rs +++ b/cli/src/cmd/forge/coverage.rs @@ -269,12 +269,7 @@ impl CoverageArgs { .sender(evm_opts.sender) .with_fork(evm_opts.get_fork(&config, env.clone())) .with_cheats_config(CheatsConfig::new(&config, &evm_opts)) - .with_test_options(TestOptions { - fuzz_runs: config.fuzz_runs, - fuzz_max_local_rejects: config.fuzz_max_local_rejects, - fuzz_max_global_rejects: config.fuzz_max_global_rejects, - ..Default::default() - }) + .with_test_options(TestOptions { fuzz: config.fuzz, ..Default::default() }) .set_coverage(true) .build(root.clone(), output, env, evm_opts)?; diff --git a/cli/src/cmd/forge/test/mod.rs b/cli/src/cmd/forge/test/mod.rs index 911a2f60f498b..1027aec6a21a2 100644 --- a/cli/src/cmd/forge/test/mod.rs +++ b/cli/src/cmd/forge/test/mod.rs @@ -311,16 +311,7 @@ pub fn custom_run(args: TestArgs) -> eyre::Result { // Merge all configs let (config, mut evm_opts) = args.load_config_and_evm_opts_emit_warnings()?; - let test_options = TestOptions { - fuzz_runs: config.fuzz_runs, - fuzz_max_local_rejects: config.fuzz_max_local_rejects, - fuzz_max_global_rejects: config.fuzz_max_global_rejects, - fuzz_seed: config.fuzz_seed, - invariant_runs: config.invariant_runs, - invariant_depth: config.invariant_depth, - invariant_fail_on_revert: config.invariant_fail_on_revert, - invariant_call_override: config.invariant_call_override, - }; + let test_options = TestOptions { fuzz: config.fuzz, invariant: config.invariant }; let mut filter = args.filter(&config); diff --git a/cli/tests/it/config.rs b/cli/tests/it/config.rs index ced024fc7d17b..8870433e7b484 100644 --- a/cli/tests/it/config.rs +++ b/cli/tests/it/config.rs @@ -13,7 +13,7 @@ use foundry_cli_test_utils::{ }; use foundry_config::{ cache::{CachedChains, CachedEndpoints, StorageCachingConfig}, - Config, OptimizerDetails, SolcReq, + Config, FuzzConfig, InvariantConfig, OptimizerDetails, SolcReq, }; use path_slash::PathBufExt; use std::{fs, path::PathBuf, str::FromStr}; @@ -57,14 +57,14 @@ forgetest!(can_extract_config_values, |prj: TestProject, mut cmd: TestCommand| { contract_pattern_inverse: None, path_pattern: None, path_pattern_inverse: None, - fuzz_runs: 1000, - fuzz_max_local_rejects: 2000, - fuzz_max_global_rejects: 100203, - fuzz_seed: Some(1000.into()), - invariant_runs: 256, - invariant_depth: 15, - invariant_fail_on_revert: false, - invariant_call_override: false, + fuzz: FuzzConfig { + runs: 1000, + max_local_rejects: 2000, + max_global_rejects: 100203, + seed: Some(1000.into()), + ..Default::default() + }, + invariant: InvariantConfig { runs: 256, ..Default::default() }, ffi: true, sender: "00a329c0648769A73afAc7F9381D08FB43dBEA72".parse().unwrap(), tx_origin: "00a329c0648769A73afAc7F9F81E08FB43dBEA72".parse().unwrap(), diff --git a/config/README.md b/config/README.md index a5ef21e5a9f5f..bd3221b32979b 100644 --- a/config/README.md +++ b/config/README.md @@ -109,11 +109,6 @@ match_contract = "Foo" no_match_contract = "Bar" match_path = "*/Foo*" no_match_path = "*/Bar*" -fuzz_runs = 256 -invariant_runs = 256 -invariant_depth = 15 -invariant_fail_on_revert = false -invariant_call_override = false ffi = false sender = '0x00a329c0648769a73afac7f9381e08fb43dbea72' tx_origin = '0x00a329c0648769a73afac7f9381e08fb43dbea72' @@ -132,9 +127,6 @@ block_gas_limit = 30000000 memory_limit = 33554432 extra_output = ["metadata"] extra_output_files = [] -fuzz_max_local_rejects = 1024 -fuzz_max_global_rejects = 65536 -fuzz_seed = '0x3e8' names = false sizes = false via_ir = false @@ -160,8 +152,30 @@ revert_strings = "default" sparse_mode = false build_info = true build_info_path = "build-info" -fmt = { line_length = 100, tab_width = 2, bracket_spacing = true } root = "root" + +[fuzz] +runs = 256 +max_local_rejects = 1024 +max_global_rejects = 65536 +seed = '0x3e8' +dictionary_weight = 40 +include_storage = true +include_push_bytes = true + +[invariant] +runs = 256 +depth = 15 +fail_on_revert = false +call_override = false +dictionary_weight = 80 +include_storage = true +include_push_bytes = true + +[fmt] +line_length = 100 +tab_width = 2 +bracket_spacing = true ``` #### Additional Optimizer settings diff --git a/config/src/fuzz.rs b/config/src/fuzz.rs new file mode 100644 index 0000000000000..dfe57be04e3a8 --- /dev/null +++ b/config/src/fuzz.rs @@ -0,0 +1,45 @@ +//! Configuration for fuzz testing + +use ethers_core::types::U256; +use serde::{Deserialize, Serialize}; + +/// Contains for fuzz testing +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct FuzzConfig { + /// The number of test cases that must execute for each property test + pub runs: u32, + /// The maximum number of local test case rejections allowed + /// by proptest, to be encountered during usage of `vm.assume` + /// cheatcode. + pub max_local_rejects: u32, + /// The maximum number of global test case rejections allowed + /// by proptest, to be encountered during usage of `vm.assume` + /// cheatcode. + pub max_global_rejects: u32, + /// Optional seed for the fuzzing RNG algorithm + #[serde( + deserialize_with = "ethers_core::types::serde_helpers::deserialize_stringified_numeric_opt" + )] + pub seed: Option, + /// The weight of the dictionary + #[serde(deserialize_with = "crate::deserialize_stringified_percent")] + pub dictionary_weight: u32, + /// The flag indicating whether to include values from storage + pub include_storage: bool, + /// The flag indicating whether to include push bytes values + pub include_push_bytes: bool, +} + +impl Default for FuzzConfig { + fn default() -> Self { + FuzzConfig { + runs: 256, + max_local_rejects: 1024, + max_global_rejects: 65536, + seed: None, + dictionary_weight: 40, + include_storage: true, + include_push_bytes: true, + } + } +} diff --git a/config/src/invariant.rs b/config/src/invariant.rs new file mode 100644 index 0000000000000..5902455dce87e --- /dev/null +++ b/config/src/invariant.rs @@ -0,0 +1,38 @@ +//! Configuration for invariant testing + +use serde::{Deserialize, Serialize}; + +/// Contains for invariant testing +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct InvariantConfig { + /// The number of runs that must execute for each invariant test group. + pub runs: u32, + /// The number of calls executed to attempt to break invariants in one run. + pub depth: u32, + /// Fails the invariant fuzzing if a revert occurs + pub fail_on_revert: bool, + /// Allows overriding an unsafe external call when running invariant tests. eg. reentrancy + /// checks + pub call_override: bool, + /// The weight of the dictionary + #[serde(deserialize_with = "crate::deserialize_stringified_percent")] + pub dictionary_weight: u32, + /// The flag indicating whether to include values from storage + pub include_storage: bool, + /// The flag indicating whether to include push bytes values + pub include_push_bytes: bool, +} + +impl Default for InvariantConfig { + fn default() -> Self { + InvariantConfig { + runs: 256, + depth: 15, + fail_on_revert: false, + call_override: false, + dictionary_weight: 80, + include_storage: true, + include_push_bytes: true, + } + } +} diff --git a/config/src/lib.rs b/config/src/lib.rs index 4a149c3a357bd..310d1bf07def5 100644 --- a/config/src/lib.rs +++ b/config/src/lib.rs @@ -21,6 +21,7 @@ use figment::{ Error, Figment, Metadata, Profile, Provider, }; use inflector::Inflector; +use once_cell::sync::Lazy; use regex::Regex; use semver::Version; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -74,6 +75,12 @@ use crate::{ }; use providers::*; +mod fuzz; +pub use fuzz::FuzzConfig; + +mod invariant; +pub use invariant::InvariantConfig; + /// Foundry configuration /// /// # Defaults @@ -204,17 +211,10 @@ pub struct Config { /// Only run tests in source files that do not match the specified glob pattern. #[serde(rename = "no_match_path", with = "from_opt_glob")] pub path_pattern_inverse: Option, - /// The number of test cases that must execute for each property test - pub fuzz_runs: u32, - /// The number of runs that must execute for each invariant test group. - pub invariant_runs: u32, - /// The number of calls executed to attempt to break invariants in one run. - pub invariant_depth: u32, - /// Fails the invariant fuzzing if a revert occurs - pub invariant_fail_on_revert: bool, - /// Allows overriding an unsafe external call when running invariant tests. eg. reentrancy - /// checks - pub invariant_call_override: bool, + /// Configuration for fuzz testing + pub fuzz: FuzzConfig, + /// Configuration for invariant testing + pub invariant: InvariantConfig, /// Whether to allow ffi cheatcodes in test pub ffi: bool, /// The address which will be executing all tests @@ -274,19 +274,6 @@ pub struct Config { /// output selection as separate files. #[serde(default)] pub extra_output_files: Vec, - /// The maximum number of local test case rejections allowed - /// by proptest, to be encountered during usage of `vm.assume` - /// cheatcode. - pub fuzz_max_local_rejects: u32, - /// The maximum number of global test case rejections allowed - /// by proptest, to be encountered during usage of `vm.assume` - /// cheatcode. - pub fuzz_max_global_rejects: u32, - /// Optional seed for the fuzzing RNG algorithm - #[serde( - deserialize_with = "ethers_core::types::serde_helpers::deserialize_stringified_numeric_opt" - )] - pub fuzz_seed: Option, /// Print the names of the compiled contracts pub names: bool, /// Print the sizes of the compiled contracts @@ -351,6 +338,10 @@ pub struct Config { pub __warnings: Vec, } +/// Mapping of fallback standalone sections. See [`FallbackProfileProvider`] +pub static STANDALONE_FALLBACK_SECTIONS: Lazy> = + Lazy::new(|| HashMap::from([("invariant", "fuzz")])); + impl Config { /// The default profile: "default" pub const DEFAULT_PROFILE: Profile = Profile::const_new("default"); @@ -362,7 +353,8 @@ impl Config { pub const PROFILE_SECTION: &'static str = "profile"; /// Standalone sections in the config which get integrated into the selected profile - pub const STANDALONE_SECTIONS: &'static [&'static str] = &["rpc_endpoints", "etherscan", "fmt"]; + pub const STANDALONE_SECTIONS: &'static [&'static str] = + &["rpc_endpoints", "etherscan", "fmt", "fuzz", "invariant"]; /// File name of config toml file pub const FILE_NAME: &'static str = "foundry.toml"; @@ -1328,7 +1320,16 @@ impl Config { } // merge special keys into config for standalone_key in Config::STANDALONE_SECTIONS { - figment = figment.merge(provider.wrap(profile.clone(), standalone_key)); + // let standalone_provider = + if let Some(fallback) = STANDALONE_FALLBACK_SECTIONS.get(standalone_key) { + figment = figment.merge( + provider + .fallback(standalone_key, fallback) + .wrap(profile.clone(), standalone_key), + ); + } else { + figment = figment.merge(provider.wrap(profile.clone(), standalone_key)); + } } // merge the profile figment = figment.merge(provider); @@ -1360,7 +1361,7 @@ impl From for Figment { // merge environment variables figment = figment .merge(Env::prefixed("DAPP_").ignore(&["REMAPPINGS", "LIBRARIES"]).global()) - .merge(Env::prefixed("DAPP_TEST_").ignore(&["CACHE"]).global()) + .merge(Env::prefixed("DAPP_TEST_").ignore(&["CACHE", "FUZZ_RUNS", "DEPTH"]).global()) .merge(DappEnvCompatProvider) .merge(Env::raw().only(&["ETHERSCAN_API_KEY"])) .merge( @@ -1542,14 +1543,8 @@ impl Default for Config { contract_pattern_inverse: None, path_pattern: None, path_pattern_inverse: None, - fuzz_runs: 256, - fuzz_max_local_rejects: 1024, - fuzz_max_global_rejects: 65536, - fuzz_seed: None, - invariant_runs: 256, - invariant_depth: 15, - invariant_fail_on_revert: false, - invariant_call_override: false, + fuzz: Default::default(), + invariant: Default::default(), ffi: false, sender: Config::DEFAULT_SENDER, tx_origin: Config::DEFAULT_SENDER, @@ -1912,6 +1907,24 @@ impl Provider for DappEnvCompatProvider { dict.insert("libraries".to_string(), utils::to_array_value(&val)?); } + let mut fuzz_dict = Dict::new(); + if let Ok(val) = env::var("DAPP_TEST_FUZZ_RUNS") { + fuzz_dict.insert( + "runs".to_string(), + val.parse::().map_err(figment::Error::custom)?.into(), + ); + } + dict.insert("fuzz".to_string(), fuzz_dict.into()); + + let mut invariant_dict = Dict::new(); + if let Ok(val) = env::var("DAPP_TEST_DEPTH") { + invariant_dict.insert( + "depth".to_string(), + val.parse::().map_err(figment::Error::custom)?.into(), + ); + } + dict.insert("invariant".to_string(), invariant_dict.into()); + Ok(Map::from([(Config::selected_profile(), dict)])) } } @@ -2320,6 +2333,13 @@ trait ProviderExt: Provider { ) -> OptionalStrictProfileProvider<&Self> { OptionalStrictProfileProvider::new(self, profiles) } + fn fallback( + &self, + profile: impl Into, + fallback: impl Into, + ) -> FallbackProfileProvider<&Self> { + FallbackProfileProvider::new(self, profile, fallback) + } } impl ProviderExt for P {} @@ -2998,14 +3018,6 @@ mod tests { extra_output_files = [] ffi = false force = false - fuzz_max_global_rejects = 65536 - fuzz_max_local_rejects = 1024 - fuzz_runs = 256 - fuzz_seed = '0x3e8' - invariant_runs = 256 - invariant_depth = 15 - invariant_fail_on_revert = false - invariant_call_override = false gas_limit = 9223372036854775807 gas_price = 0 gas_reports = ['*'] @@ -3039,12 +3051,23 @@ mod tests { mainnet = "${RPC_MAINNET}" mainnet_2 = "https://eth-mainnet.alchemyapi.io/v2/${API_KEY}" + [fuzz] + runs = 256 + seed = '0x3e8' + max_global_rejects = 65536 + max_local_rejects = 1024 + + [invariant] + runs = 256 + depth = 15 + fail_on_revert = false + call_override = false "#, )?; let config = Config::load_with_root(jail.directory()); - assert_eq!(config.fuzz_seed, Some(1000.into())); + assert_eq!(config.fuzz.seed, Some(1000.into())); assert_eq!( config.remappings, vec![Remapping::from_str("nested/=lib/nested/").unwrap().into()] @@ -3259,6 +3282,54 @@ mod tests { }); } + #[test] + #[should_panic] + fn test_parse_invalid_fuzz_weight() { + figment::Jail::expect_with(|jail| { + jail.create_file( + "foundry.toml", + r#" + [fuzz] + dictionary_weight = 101 + "#, + )?; + let _config = Config::load(); + Ok(()) + }); + } + + #[test] + fn test_fallback_provider() { + figment::Jail::expect_with(|jail| { + jail.create_file( + "foundry.toml", + r#" + [fuzz] + runs = 1 + include_storage = false + dictionary_weight = 99 + + [invariant] + runs = 420 + "#, + )?; + + let invariant_default = InvariantConfig::default(); + let config = Config::load(); + + assert_ne!(config.invariant.runs, config.fuzz.runs); + assert_eq!(config.invariant.runs, 420); + + assert_ne!(config.fuzz.include_storage, invariant_default.include_storage); + assert_eq!(config.invariant.include_storage, config.fuzz.include_storage); + + assert_ne!(config.fuzz.dictionary_weight, invariant_default.dictionary_weight); + assert_eq!(config.invariant.dictionary_weight, config.fuzz.dictionary_weight); + + Ok(()) + }) + } + #[test] fn can_handle_deviating_dapp_aliases() { figment::Jail::expect_with(|jail| { @@ -3266,6 +3337,7 @@ mod tests { jail.set_env("DAPP_TEST_NUMBER", 1337); jail.set_env("DAPP_TEST_ADDRESS", format!("{:?}", addr)); jail.set_env("DAPP_TEST_FUZZ_RUNS", 420); + jail.set_env("DAPP_TEST_DEPTH", 20); jail.set_env("DAPP_FORK_BLOCK", 100); jail.set_env("DAPP_BUILD_OPTIMIZE_RUNS", 999); jail.set_env("DAPP_BUILD_OPTIMIZE", 0); @@ -3274,7 +3346,8 @@ mod tests { assert_eq!(config.block_number, 1337); assert_eq!(config.sender, addr); - assert_eq!(config.fuzz_runs, 420); + assert_eq!(config.fuzz.runs, 420); + assert_eq!(config.invariant.depth, 20); assert_eq!(config.fork_block_number, Some(100)); assert_eq!(config.optimizer_runs, 999); assert!(!config.optimizer); @@ -3568,6 +3641,28 @@ mod tests { }); } + #[test] + fn test_invariant_config() { + figment::Jail::expect_with(|jail| { + jail.create_file( + "foundry.toml", + r#" + [invariant] + runs = 512 + depth = 10 + "#, + )?; + + let loaded = Config::load().sanitized(); + assert_eq!( + loaded.invariant, + InvariantConfig { runs: 512, depth: 10, ..Default::default() } + ); + + Ok(()) + }); + } + #[test] fn test_parse_with_profile() { let foundry_str = r#" diff --git a/config/src/providers.rs b/config/src/providers.rs index 07f45ee0abeba..d788d767b4eeb 100644 --- a/config/src/providers.rs +++ b/config/src/providers.rs @@ -77,3 +77,41 @@ impl Provider for WarningsProvider

{ Some(self.profile.clone()) } } + +/// Extracts the profile from the `profile` key and sets unset values according to the fallback +/// provider +pub struct FallbackProfileProvider

{ + provider: P, + profile: Profile, + fallback: Profile, +} + +impl

FallbackProfileProvider

{ + pub fn new(provider: P, profile: impl Into, fallback: impl Into) -> Self { + FallbackProfileProvider { provider, profile: profile.into(), fallback: fallback.into() } + } +} + +impl Provider for FallbackProfileProvider

{ + fn metadata(&self) -> Metadata { + self.provider.metadata() + } + + fn data(&self) -> Result, Error> { + if let Some(fallback) = self.provider.data()?.get(&self.fallback) { + let mut inner = self.provider.data()?.remove(&self.profile).unwrap_or_default(); + for (k, v) in fallback.iter() { + if !inner.contains_key(k) { + inner.insert(k.to_owned(), v.clone()); + } + } + Ok(self.profile.collect(inner)) + } else { + self.provider.data() + } + } + + fn profile(&self) -> Option { + Some(self.profile.clone()) + } +} diff --git a/config/src/utils.rs b/config/src/utils.rs index 5a706312af35c..5cce060830654 100644 --- a/config/src/utils.rs +++ b/config/src/utils.rs @@ -1,8 +1,10 @@ //! Utility functions use crate::Config; +use ethers_core::types::{serde_helpers::Numeric, U256}; use ethers_solc::remappings::{Remapping, RemappingError}; use figment::value::Value; +use serde::{Deserialize, Deserializer}; use std::{ path::{Path, PathBuf}, str::FromStr, @@ -165,3 +167,18 @@ pub(crate) fn get_dir_remapping(dir: impl AsRef) -> Option { None } } + +/// Deserialize stringified percent. The value must be between 0 and 100 inclusive. +pub(crate) fn deserialize_stringified_percent<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let num: U256 = + Numeric::deserialize(deserializer)?.try_into().map_err(serde::de::Error::custom)?; + let num: u64 = num.try_into().map_err(serde::de::Error::custom)?; + if num <= 100 { + num.try_into().map_err(serde::de::Error::custom) + } else { + Err(serde::de::Error::custom("percent must be lte 100")) + } +} diff --git a/evm/src/fuzz/invariant/executor.rs b/evm/src/fuzz/invariant/executor.rs index a21901bc39365..f7c56122490f1 100644 --- a/evm/src/fuzz/invariant/executor.rs +++ b/evm/src/fuzz/invariant/executor.rs @@ -2,7 +2,7 @@ use super::{ assert_invariants, filters::{ArtifactFilters, SenderFilters}, BasicTxDetails, FuzzRunIdentifiedContracts, InvariantContract, InvariantFuzzError, - InvariantFuzzTestResult, InvariantTestOptions, RandomCallGenerator, TargetedContracts, + InvariantFuzzTestResult, RandomCallGenerator, TargetedContracts, }; use crate::{ executor::{ @@ -24,6 +24,7 @@ use ethers::{ }; use eyre::ContextCompat; use foundry_common::contracts::{ContractsByAddress, ContractsByArtifact}; +use foundry_config::InvariantConfig; use hashbrown::HashMap; use parking_lot::{Mutex, RwLock}; use proptest::{ @@ -47,6 +48,8 @@ pub struct InvariantExecutor<'a> { pub executor: &'a mut Executor, /// Proptest runner. runner: TestRunner, + /// The invariant configuration + config: InvariantConfig, /// Contracts deployed with `setUp()` setup_contracts: &'a ContractsByAddress, /// Contracts that are part of the project but have not been deployed yet. We need the bytecode @@ -61,12 +64,14 @@ impl<'a> InvariantExecutor<'a> { pub fn new( executor: &'a mut Executor, runner: TestRunner, + config: InvariantConfig, setup_contracts: &'a ContractsByAddress, project_contracts: &'a ContractsByArtifact, ) -> Self { Self { executor, runner, + config, setup_contracts, project_contracts, artifact_filters: ArtifactFilters::default(), @@ -78,10 +83,8 @@ impl<'a> InvariantExecutor<'a> { pub fn invariant_fuzz( &mut self, invariant_contract: InvariantContract, - test_options: InvariantTestOptions, ) -> eyre::Result> { - let (fuzz_state, targeted_contracts, strat) = - self.prepare_fuzzing(&invariant_contract, test_options)?; + let (fuzz_state, targeted_contracts, strat) = self.prepare_fuzzing(&invariant_contract)?; // Stores the consumed gas and calldata of every successful fuzz call. let fuzz_cases: RefCell> = RefCell::new(Default::default()); @@ -114,7 +117,7 @@ impl<'a> InvariantExecutor<'a> { let _ = self.runner.run(&strat, |mut inputs| { // Scenarios where we want to fail as soon as possible. { - if test_options.fail_on_revert && failures.borrow().reverts == 1 { + if self.config.fail_on_revert && failures.borrow().reverts == 1 { return Err(TestCaseError::fail("Revert occurred.")) } @@ -129,12 +132,12 @@ impl<'a> InvariantExecutor<'a> { let mut executor = blank_executor.borrow().clone(); // Used for stat reports (eg. gas usage). - let mut fuzz_runs = Vec::with_capacity(test_options.depth as usize); + let mut fuzz_runs = Vec::with_capacity(self.config.depth as usize); // Created contracts during a run. let mut created_contracts = vec![]; - 'fuzz_run: for _ in 0..test_options.depth { + 'fuzz_run: for _ in 0..self.config.depth { let (sender, (address, calldata)) = inputs.last().expect("to have the next randomly generated input."); @@ -147,7 +150,14 @@ impl<'a> InvariantExecutor<'a> { let mut state_changeset = call_result.state_changeset.to_owned().expect("to have a state changeset."); - collect_data(&mut state_changeset, sender, &call_result, fuzz_state.clone()); + collect_data( + &mut state_changeset, + sender, + &call_result, + fuzz_state.clone(), + self.config.include_storage, + self.config.include_push_bytes, + ); if let Err(error) = collect_created_contracts( &state_changeset, @@ -175,7 +185,7 @@ impl<'a> InvariantExecutor<'a> { &executor, &inputs, &mut failures.borrow_mut(), - test_options, + self.config.fail_on_revert, ) { break 'fuzz_run } @@ -218,7 +228,6 @@ impl<'a> InvariantExecutor<'a> { fn prepare_fuzzing( &mut self, invariant_contract: &InvariantContract, - test_options: InvariantTestOptions, ) -> eyre::Result { // Finds out the chosen deployed contracts and/or senders. self.select_contract_artifacts(invariant_contract.address, invariant_contract.abi)?; @@ -230,7 +239,8 @@ impl<'a> InvariantExecutor<'a> { } // Stores fuzz state for use with [fuzz_calldata_from_state]. - let fuzz_state: EvmFuzzState = build_initial_state(self.executor.backend().mem_db()); + let fuzz_state: EvmFuzzState = + build_initial_state(self.executor.backend().mem_db(), self.config.include_storage); // During execution, any newly created contract is added here and used through the rest of // the fuzz run. @@ -238,15 +248,19 @@ impl<'a> InvariantExecutor<'a> { Arc::new(Mutex::new(targeted_contracts)); // Creates the invariant strategy. - let strat = - invariant_strat(fuzz_state.clone(), targeted_senders, targeted_contracts.clone()) - .no_shrink() - .boxed(); + let strat = invariant_strat( + fuzz_state.clone(), + targeted_senders, + targeted_contracts.clone(), + self.config.dictionary_weight, + ) + .no_shrink() + .boxed(); // Allows `override_call_strat` to use the address given by the Fuzzer inspector during // EVM execution. let mut call_generator = None; - if test_options.call_override { + if self.config.call_override { let target_contract_ref = Arc::new(RwLock::new(Address::zero())); call_generator = Some(RandomCallGenerator::new( @@ -507,6 +521,8 @@ fn collect_data( sender: &Address, call_result: &RawCallResult, fuzz_state: EvmFuzzState, + include_storage: bool, + include_push_bytes: bool, ) { // Verify it has no code. let mut has_code = false; @@ -521,7 +537,13 @@ fn collect_data( sender_changeset = state_changeset.remove(sender); } - collect_state_from_call(&call_result.logs, &*state_changeset, fuzz_state); + collect_state_from_call( + &call_result.logs, + &*state_changeset, + fuzz_state, + include_storage, + include_push_bytes, + ); // Re-add changes if let Some(changed) = sender_changeset { @@ -536,7 +558,7 @@ fn can_continue( executor: &Executor, calldata: &[BasicTxDetails], failures: &mut InvariantFailures, - test_options: InvariantTestOptions, + fail_on_revert: bool, ) -> bool { if !call_result.reverted { if assert_invariants(invariant_contract, executor, calldata, failures).is_err() { @@ -547,7 +569,7 @@ fn can_continue( // The user might want to stop all execution if a revert happens to // better bound their testing space. - if test_options.fail_on_revert { + if fail_on_revert { let error = InvariantFuzzError::new(invariant_contract, None, calldata, call_result, &[]); diff --git a/evm/src/fuzz/invariant/mod.rs b/evm/src/fuzz/invariant/mod.rs index 215dc0208e02d..42f70bb6f1edc 100644 --- a/evm/src/fuzz/invariant/mod.rs +++ b/evm/src/fuzz/invariant/mod.rs @@ -34,18 +34,6 @@ pub struct InvariantContract<'a> { pub abi: &'a Abi, } -/// Metadata on how to run invariant tests -#[derive(Debug, Clone, Copy, Default)] -pub struct InvariantTestOptions { - /// The number of calls executed to attempt to break invariants in one run. - pub depth: u32, - /// Fails the invariant fuzzing if a revert occurs - pub fail_on_revert: bool, - /// Allows overriding an unsafe external call when running invariant tests. eg. reetrancy - /// checks - pub call_override: bool, -} - /// Given the executor state, asserts that no invariant has been broken. Otherwise, it fills the /// external `invariant_failures.failed_invariant` map and returns a generic error. pub fn assert_invariants( diff --git a/evm/src/fuzz/mod.rs b/evm/src/fuzz/mod.rs index d06d35ff12e02..153471d54d4ce 100644 --- a/evm/src/fuzz/mod.rs +++ b/evm/src/fuzz/mod.rs @@ -8,7 +8,8 @@ use ethers::{ types::{Address, Bytes, Log}, }; use foundry_common::{calc, contracts::ContractsByAddress}; -pub use proptest::test_runner::{Config as FuzzConfig, Reason}; +use foundry_config::FuzzConfig; +pub use proptest::test_runner::Reason; use proptest::test_runner::{TestCaseError, TestError, TestRunner}; use serde::{Deserialize, Serialize}; use std::{cell::RefCell, collections::BTreeMap, fmt}; @@ -34,12 +35,19 @@ pub struct FuzzedExecutor<'a> { runner: TestRunner, /// The account that calls tests sender: Address, + /// The fuzz configuration + config: FuzzConfig, } impl<'a> FuzzedExecutor<'a> { /// Instantiates a fuzzed executor given a testrunner - pub fn new(executor: &'a Executor, runner: TestRunner, sender: Address) -> Self { - Self { executor, runner, sender } + pub fn new( + executor: &'a Executor, + runner: TestRunner, + sender: Address, + config: FuzzConfig, + ) -> Self { + Self { executor, runner, sender, config } } /// Fuzzes the provided function, assuming it is available at the contract at `address` @@ -65,16 +73,14 @@ impl<'a> FuzzedExecutor<'a> { // Stores fuzz state for use with [fuzz_calldata_from_state] let state: EvmFuzzState = if let Some(fork_db) = self.executor.backend().active_fork_db() { - build_initial_state(fork_db) + build_initial_state(fork_db, self.config.include_storage) } else { - build_initial_state(self.executor.backend().mem_db()) + build_initial_state(self.executor.backend().mem_db(), self.config.include_storage) }; - // TODO: We should have a `FuzzerOpts` struct where we can configure the fuzzer. When we - // have that, we should add a way to configure strategy weights let strat = proptest::strategy::Union::new_weighted(vec![ - (60, fuzz_calldata(func.clone())), - (40, fuzz_calldata_from_state(func.clone(), state.clone())), + (100 - self.config.dictionary_weight, fuzz_calldata(func.clone())), + (self.config.dictionary_weight, fuzz_calldata_from_state(func.clone(), state.clone())), ]); tracing::debug!(func = ?func.name, should_fail, "fuzzing"); let run_result = self.runner.clone().run(&strat, |calldata| { @@ -86,7 +92,13 @@ impl<'a> FuzzedExecutor<'a> { call.state_changeset.as_ref().expect("We should have a state changeset."); // Build fuzzer state - collect_state_from_call(&call.logs, state_changeset, state.clone()); + collect_state_from_call( + &call.logs, + state_changeset, + state.clone(), + self.config.include_storage, + self.config.include_push_bytes, + ); // When assume cheat code is triggered return a special string "FOUNDRY::ASSUME" if call.result.as_ref() == ASSUME_MAGIC_RETURN_CODE { diff --git a/evm/src/fuzz/strategies/invariants.rs b/evm/src/fuzz/strategies/invariants.rs index a93481df61ef4..28bdfc373ec36 100644 --- a/evm/src/fuzz/strategies/invariants.rs +++ b/evm/src/fuzz/strategies/invariants.rs @@ -57,10 +57,11 @@ pub fn invariant_strat( fuzz_state: EvmFuzzState, senders: SenderFilters, contracts: FuzzRunIdentifiedContracts, + dictionary_weight: u32, ) -> BoxedStrategy> { // We only want to seed the first value, since we want to generate the rest as we mutate the // state - vec![generate_call(fuzz_state, senders, contracts); 1].boxed() + vec![generate_call(fuzz_state, senders, contracts, dictionary_weight); 1].boxed() } /// Strategy to generate a transaction where the `sender`, `target` and `calldata` are all generated @@ -69,6 +70,7 @@ fn generate_call( fuzz_state: EvmFuzzState, senders: SenderFilters, contracts: FuzzRunIdentifiedContracts, + dictionary_weight: u32, ) -> BoxedStrategy { let random_contract = select_random_contract(contracts); let senders = Rc::new(senders); @@ -78,7 +80,8 @@ fn generate_call( let senders = senders.clone(); let fuzz_state = fuzz_state.clone(); func.prop_flat_map(move |func| { - let sender = select_random_sender(fuzz_state.clone(), senders.clone()); + let sender = + select_random_sender(fuzz_state.clone(), senders.clone(), dictionary_weight); (sender, fuzz_contract_with_calldata(fuzz_state.clone(), contract, func)) }) }) @@ -92,17 +95,18 @@ fn generate_call( fn select_random_sender( fuzz_state: EvmFuzzState, senders: Rc, + dictionary_weight: u32, ) -> impl Strategy { let senders_ref = senders.clone(); let fuzz_strategy = proptest::strategy::Union::new_weighted(vec![ ( - 10, + 100 - dictionary_weight, fuzz_param(&ParamType::Address) .prop_map(move |addr| addr.into_address().unwrap()) .boxed(), ), ( - 90, + dictionary_weight, fuzz_param_from_state(&ParamType::Address, fuzz_state) .prop_map(move |addr| addr.into_address().unwrap()) .boxed(), @@ -178,8 +182,8 @@ pub fn fuzz_contract_with_calldata( contract: Address, func: Function, ) -> impl Strategy { - // // We need to compose all the strategies generated for each parameter in all - // // possible combinations + // We need to compose all the strategies generated for each parameter in all + // possible combinations let strats = proptest::strategy::Union::new_weighted(vec![ (60, fuzz_calldata(func.clone())), (40, fuzz_calldata_from_state(func, fuzz_state)), diff --git a/evm/src/fuzz/strategies/param.rs b/evm/src/fuzz/strategies/param.rs index 3c77971b829e0..34391bbaeaa5c 100644 --- a/evm/src/fuzz/strategies/param.rs +++ b/evm/src/fuzz/strategies/param.rs @@ -142,7 +142,7 @@ mod tests { let func = HumanReadableParser::parse_function(f).unwrap(); let db = CacheDB::new(EmptyDB()); - let state = build_initial_state(&db); + let state = build_initial_state(&db, true); let strat = proptest::strategy::Union::new_weighted(vec![ (60, fuzz_calldata(func.clone())), diff --git a/evm/src/fuzz/strategies/state.rs b/evm/src/fuzz/strategies/state.rs index 7e79b91eccf60..772f1a247a72a 100644 --- a/evm/src/fuzz/strategies/state.rs +++ b/evm/src/fuzz/strategies/state.rs @@ -80,17 +80,22 @@ This is a bug, please open an issue: https://github.com/foundry-rs/foundry/issue } /// Builds the initial [EvmFuzzState] from a database. -pub fn build_initial_state(db: &CacheDB) -> EvmFuzzState { +pub fn build_initial_state( + db: &CacheDB, + include_storage: bool, +) -> EvmFuzzState { let mut state = FuzzDictionary::default(); for (address, account) in db.accounts.iter() { // Insert basic account information state.insert(H256::from(*address).into()); - // Insert storage - for (slot, value) in &account.storage { - state.insert(utils::u256_to_h256_be(*slot).into()); - state.insert(utils::u256_to_h256_be(*value).into()); + if include_storage { + // Insert storage + for (slot, value) in &account.storage { + state.insert(utils::u256_to_h256_be(*slot).into()); + state.insert(utils::u256_to_h256_be(*value).into()); + } } } @@ -109,6 +114,8 @@ pub fn collect_state_from_call( logs: &[Log], state_changeset: &StateChangeset, state: EvmFuzzState, + include_storage: bool, + include_push_bytes: bool, ) { let mut state = state.write(); @@ -116,19 +123,23 @@ pub fn collect_state_from_call( // Insert basic account information state.insert(H256::from(*address).into()); - // Insert storage - for (slot, value) in &account.storage { - state.insert(utils::u256_to_h256_be(*slot).into()); - state.insert(utils::u256_to_h256_be(value.present_value()).into()); + if include_storage { + // Insert storage + for (slot, value) in &account.storage { + state.insert(utils::u256_to_h256_be(*slot).into()); + state.insert(utils::u256_to_h256_be(value.present_value()).into()); + } } - // Insert push bytes - if let Some(code) = &account.info.code { - if !state.cache.contains(address) { - state.cache.insert(*address); + if include_push_bytes { + // Insert push bytes + if let Some(code) = &account.info.code { + if !state.cache.contains(address) { + state.cache.insert(*address); - for push_byte in collect_push_bytes(code.bytes().clone()) { - state.insert(push_byte); + for push_byte in collect_push_bytes(code.bytes().clone()) { + state.insert(push_byte); + } } } } diff --git a/forge/Cargo.toml b/forge/Cargo.toml index 1d44a80c6f833..733f775619a7a 100644 --- a/forge/Cargo.toml +++ b/forge/Cargo.toml @@ -33,3 +33,4 @@ parking_lot = "0.12" [dev-dependencies] ethers = { git = "https://github.com/gakonst/ethers-rs", default-features = false, features = ["solc-full", "solc-tests"] } foundry-utils = { path = "./../utils", features = ["test"] } + diff --git a/forge/src/lib.rs b/forge/src/lib.rs index ec47b12051d91..6ab228465e4dd 100644 --- a/forge/src/lib.rs +++ b/forge/src/lib.rs @@ -9,7 +9,6 @@ pub mod coverage; /// The Forge test runner mod runner; -use ethers::types::U256; pub use runner::ContractRunner; /// Forge test runners for multiple contracts @@ -27,36 +26,19 @@ pub use foundry_evm::*; /// Metadata on how to run fuzz/invariant tests #[derive(Debug, Clone, Copy, Default)] pub struct TestOptions { - /// The number of test cases that must execute for each fuzz test - pub fuzz_runs: u32, - /// The maximum number of global test case rejections allowed - /// by proptest, to be encountered during usage of `vm.assume` - /// cheatcode. - pub fuzz_max_local_rejects: u32, - /// The maximum number of local test case rejections allowed - /// by proptest, to be encountered during usage of `vm.assume` - /// cheatcode. - pub fuzz_max_global_rejects: u32, - /// Optional seed for the fuzzing RNG algorithm - pub fuzz_seed: Option, - /// The number of runs that must execute for each invariant test group. - pub invariant_runs: u32, - /// The number of calls executed to attempt to break invariants in one run. - pub invariant_depth: u32, - /// Fails the invariant fuzzing if a revert occurs - pub invariant_fail_on_revert: bool, - /// Allows overriding an unsafe external call when running invariant tests. eg. reetrancy - /// checks - pub invariant_call_override: bool, + /// The fuzz test configuration + pub fuzz: foundry_config::FuzzConfig, + /// The invariant test configuration + pub invariant: foundry_config::InvariantConfig, } impl TestOptions { pub fn invariant_fuzzer(&self) -> TestRunner { - self.fuzzer_with_cases(self.invariant_runs) + self.fuzzer_with_cases(self.invariant.runs) } pub fn fuzzer(&self) -> TestRunner { - self.fuzzer_with_cases(self.fuzz_runs) + self.fuzzer_with_cases(self.fuzz.runs) } pub fn fuzzer_with_cases(&self, cases: u32) -> TestRunner { @@ -64,12 +46,12 @@ impl TestOptions { let cfg = proptest::test_runner::Config { failure_persistence: None, cases, - max_local_rejects: self.fuzz_max_local_rejects, - max_global_rejects: self.fuzz_max_global_rejects, + max_local_rejects: self.fuzz.max_local_rejects, + max_global_rejects: self.fuzz.max_global_rejects, ..Default::default() }; - if let Some(ref fuzz_seed) = self.fuzz_seed { + if let Some(ref fuzz_seed) = self.fuzz.seed { trace!(target: "forge::test", "building deterministic fuzzer with seed {}", fuzz_seed); let mut bytes: [u8; 32] = [0; 32]; fuzz_seed.to_big_endian(&mut bytes); diff --git a/forge/src/runner.rs b/forge/src/runner.rs index 2f49ff1c3131a..fe886678ebf05 100644 --- a/forge/src/runner.rs +++ b/forge/src/runner.rs @@ -14,9 +14,7 @@ use foundry_common::{ use foundry_evm::{ executor::{CallResult, DeployResult, EvmError, Executor}, fuzz::{ - invariant::{ - InvariantContract, InvariantExecutor, InvariantFuzzTestResult, InvariantTestOptions, - }, + invariant::{InvariantContract, InvariantExecutor, InvariantFuzzTestResult}, FuzzedExecutor, }, trace::{load_contracts, TraceKind}, @@ -443,6 +441,7 @@ impl<'a> ContractRunner<'a> { let mut evm = InvariantExecutor::new( &mut self.executor, runner, + test_options.invariant, &identified_contracts, project_contracts, ); @@ -450,14 +449,9 @@ impl<'a> ContractRunner<'a> { let invariant_contract = InvariantContract { address, invariant_functions: functions, abi: self.contract }; - if let Some(InvariantFuzzTestResult { invariants, cases, reverts }) = evm.invariant_fuzz( - invariant_contract, - InvariantTestOptions { - depth: test_options.invariant_depth, - fail_on_revert: test_options.invariant_fail_on_revert, - call_override: test_options.invariant_call_override, - }, - )? { + if let Some(InvariantFuzzTestResult { invariants, cases, reverts }) = + evm.invariant_fuzz(invariant_contract)? + { let results = invariants .iter() .map(|(_, test_error)| { @@ -510,12 +504,13 @@ impl<'a> ContractRunner<'a> { // Run fuzz test let start = Instant::now(); - let mut result = FuzzedExecutor::new(&self.executor, runner, self.sender).fuzz( - func, - address, - should_fail, - self.errors, - ); + let mut result = FuzzedExecutor::new( + &self.executor, + runner, + self.sender, + Default::default(), + ) + .fuzz(func, address, should_fail, self.errors); // Record logs, labels and traces logs.append(&mut result.logs); diff --git a/forge/tests/it/config.rs b/forge/tests/it/config.rs index b86e66b8a4a28..48ca7cfde7055 100644 --- a/forge/tests/it/config.rs +++ b/forge/tests/it/config.rs @@ -2,19 +2,29 @@ use crate::test_helpers::{COMPILED, COMPILED_WITH_LIBS, EVM_OPTS, LIBS_PROJECT, PROJECT}; use forge::{result::SuiteResult, MultiContractRunner, MultiContractRunnerBuilder, TestOptions}; -use foundry_config::{Config, RpcEndpoint, RpcEndpoints}; +use foundry_config::{Config, FuzzConfig, InvariantConfig, RpcEndpoint, RpcEndpoints}; use foundry_evm::{decode::decode_console_logs, executor::inspector::CheatsConfig}; use std::{collections::BTreeMap, path::Path}; pub static TEST_OPTS: TestOptions = TestOptions { - fuzz_runs: 256, - fuzz_max_local_rejects: 1024, - fuzz_max_global_rejects: 65536, - fuzz_seed: None, - invariant_runs: 256, - invariant_depth: 15, - invariant_fail_on_revert: false, - invariant_call_override: false, + fuzz: FuzzConfig { + runs: 256, + max_local_rejects: 1024, + max_global_rejects: 65536, + seed: None, + include_storage: true, + include_push_bytes: true, + dictionary_weight: 40, + }, + invariant: InvariantConfig { + runs: 256, + depth: 15, + dictionary_weight: 80, + fail_on_revert: false, + call_override: false, + include_storage: true, + include_push_bytes: true, + }, }; /// Builds a base runner diff --git a/forge/tests/it/fuzz.rs b/forge/tests/it/fuzz.rs index 1f6649eebf776..b68a5e702f18b 100644 --- a/forge/tests/it/fuzz.rs +++ b/forge/tests/it/fuzz.rs @@ -56,10 +56,10 @@ fn test_fuzz_collection() { let mut runner = runner(); let mut opts = TEST_OPTS; - opts.invariant_depth = 100; - opts.invariant_runs = 1000; - opts.fuzz_runs = 1000; - opts.fuzz_seed = Some(U256::from(6u32)); + opts.invariant.depth = 100; + opts.invariant.runs = 1000; + opts.fuzz.runs = 1000; + opts.fuzz.seed = Some(U256::from(6u32)); runner.test_options = opts; let results = diff --git a/forge/tests/it/invariant.rs b/forge/tests/it/invariant.rs index 14e5dce3dc387..731b39d51cdd6 100644 --- a/forge/tests/it/invariant.rs +++ b/forge/tests/it/invariant.rs @@ -80,7 +80,7 @@ fn test_invariant_override() { let mut runner = runner(); let mut opts = TEST_OPTS; - opts.invariant_call_override = true; + opts.invariant.call_override = true; runner.test_options = opts; let results = runner @@ -105,8 +105,8 @@ fn test_invariant_storage() { let mut runner = runner(); let mut opts = TEST_OPTS; - opts.invariant_depth = 100; - opts.fuzz_seed = Some(U256::from(6u32)); + opts.invariant.depth = 100; + opts.fuzz.seed = Some(U256::from(6u32)); runner.test_options = opts; let results = runner @@ -136,7 +136,7 @@ fn test_invariant_shrink() { let mut runner = runner(); let mut opts = TEST_OPTS; - opts.fuzz_seed = Some(U256::from(102u32)); + opts.fuzz.seed = Some(U256::from(102u32)); runner.test_options = opts; let results = runner diff --git a/forge/tests/it/test_helpers.rs b/forge/tests/it/test_helpers.rs index 308224a1bb2ac..c4a82f45f473a 100644 --- a/forge/tests/it/test_helpers.rs +++ b/forge/tests/it/test_helpers.rs @@ -79,7 +79,12 @@ pub static EVM_OPTS: Lazy = Lazy::new(|| EvmOpts { pub fn fuzz_executor(executor: &Executor) -> FuzzedExecutor { let cfg = proptest::test_runner::Config { failure_persistence: None, ..Default::default() }; - FuzzedExecutor::new(executor, proptest::test_runner::TestRunner::new(cfg), CALLER) + FuzzedExecutor::new( + executor, + proptest::test_runner::TestRunner::new(cfg), + CALLER, + config::TEST_OPTS.fuzz, + ) } pub const RE_PATH_SEPARATOR: &str = "/"; diff --git a/testdata/foundry.toml b/testdata/foundry.toml index 0d4d10293818e..4b21f5cf4c0db 100644 --- a/testdata/foundry.toml +++ b/testdata/foundry.toml @@ -13,9 +13,6 @@ extra_output = [] extra_output_files = [] ffi = false force = false -fuzz_max_global_rejects = 65536 -fuzz_max_local_rejects = 1024 -fuzz_runs = 256 invariant_fail_on_revert = false invariant_call_override = false gas_limit = 9223372036854775807 @@ -45,3 +42,7 @@ via_ir = false chains = 'all' endpoints = 'all' +[fuzz] +runs = 256 +max_global_rejects = 65536 +max_local_rejects = 1024