From 07468d276d1552c869e83daf1c0efbc39815402d Mon Sep 17 00:00:00 2001 From: franfran Date: Mon, 21 Aug 2023 23:15:43 +0300 Subject: [PATCH 1/7] fuzz single refactor --- crates/evm/src/fuzz/mod.rs | 182 ++++++++++++++++++++++++------------- 1 file changed, 117 insertions(+), 65 deletions(-) diff --git a/crates/evm/src/fuzz/mod.rs b/crates/evm/src/fuzz/mod.rs index 3d7f0c559aaad..572743f6769fc 100644 --- a/crates/evm/src/fuzz/mod.rs +++ b/crates/evm/src/fuzz/mod.rs @@ -1,5 +1,6 @@ use crate::{ coverage::HitMaps, + debug::DebugArena, decode::{self, decode_console_logs}, executor::{Executor, RawCallResult}, trace::CallTraceArena, @@ -10,10 +11,11 @@ use ethers::{ types::{Address, Bytes, Log}, }; use eyre::Result; -use foundry_common::{calc, contracts::ContractsByAddress}; +use foundry_common::{calc, contracts::ContractsByAddress, evm::Breakpoints}; use foundry_config::FuzzConfig; pub use proptest::test_runner::Reason; use proptest::test_runner::{TestCaseError, TestError, TestRunner}; +use revm::interpreter::InstructionResult; use serde::{Deserialize, Serialize}; use std::{cell::RefCell, collections::BTreeMap, fmt}; use strategies::{ @@ -25,6 +27,28 @@ pub mod error; pub mod invariant; pub mod strategies; +pub struct CaseOutcome { + pub case: FuzzCase, + pub gas_used: u64, + pub stipend: u64, + pub traces: Option, + pub coverage: Option, + pub debug: Option, + pub breakpoints: Breakpoints, +} + +pub struct CounterExampleOutcome { + pub counterexample: (ethers::types::Bytes, RawCallResult), + pub exit_reason: InstructionResult, + pub debug: Option, + pub breakpoints: Breakpoints, +} + +pub enum FuzzOutcome { + Case(CaseOutcome), + CounterExample(CounterExampleOutcome), +} + /// Wrapper around an [`Executor`] which provides fuzzing support using [`proptest`](https://docs.rs/proptest/1.0.0/proptest/). /// /// After instantiation, calling `fuzz` will proceed to hammer the deployed smart contract with @@ -101,72 +125,45 @@ impl<'a> FuzzedExecutor<'a> { let strat = proptest::strategy::Union::new_weighted(weights); debug!(func = ?func.name, should_fail, "fuzzing"); let run_result = self.runner.clone().run(&strat, |calldata| { - let call = self - .executor - .call_raw(self.sender, address, calldata.0.clone(), 0.into()) - .map_err(|_| TestCaseError::fail(FuzzError::FailedContractCall))?; - let state_changeset = call - .state_changeset - .as_ref() - .ok_or_else(|| TestCaseError::fail(FuzzError::EmptyChangeset))?; - - // Build fuzzer state - collect_state_from_call( - &call.logs, - state_changeset, - state.clone(), - &self.config.dictionary, - ); - - // When assume cheat code is triggered return a special string "FOUNDRY::ASSUME" - if call.result.as_ref() == ASSUME_MAGIC_RETURN_CODE { - return Err(TestCaseError::reject(FuzzError::AssumeReject)) - } - - let success = self.executor.is_success( - address, - call.reverted, - state_changeset.clone(), - should_fail, - ); - - if success { - let mut first_case = first_case.borrow_mut(); - if first_case.is_none() { - first_case.replace(FuzzCase { - calldata, - gas: call.gas_used, - stipend: call.stipend, - }); + let fuzz_res = self.single_fuzz(&state, address, should_fail, calldata)?; + + match fuzz_res { + FuzzOutcome::Case(case) => { + let mut first_case = first_case.borrow_mut(); + if first_case.is_none() { + first_case.replace(case.case); + } + gas_by_case.borrow_mut().push((case.gas_used, case.stipend)); + + traces.replace(case.traces); + + if let Some(prev) = coverage.take() { + // Safety: If `Option::or` evaluates to `Some`, then `call.coverage` must + // necessarily also be `Some` + coverage.replace(Some(prev.merge(case.coverage.unwrap()))); + } else { + coverage.replace(case.coverage); + } + + Ok(()) } - gas_by_case.borrow_mut().push((call.gas_used, call.stipend)); - - traces.replace(call.traces); - - if let Some(prev) = coverage.take() { - // Safety: If `Option::or` evaluates to `Some`, then `call.coverage` must - // necessarily also be `Some` - coverage.replace(Some(prev.merge(call.coverage.unwrap()))); - } else { - coverage.replace(call.coverage); + FuzzOutcome::CounterExample(CounterExampleOutcome { + exit_reason, + counterexample: _counterexample, + .. + }) => { + let status = exit_reason; + // We cannot use the calldata returned by the test runner in `TestError::Fail`, + // since that input represents the last run case, which may not correspond with + // our failure - when a fuzz case fails, proptest will try + // to run at least one more case to find a minimal failure + // case. + let call_res = _counterexample.1.result.clone(); + *counterexample.borrow_mut() = _counterexample; + Err(TestCaseError::fail( + decode::decode_revert(&call_res, errors, Some(status)).unwrap_or_default(), + )) } - - Ok(()) - } else { - let status = call.exit_reason; - // We cannot use the calldata returned by the test runner in `TestError::Fail`, - // since that input represents the last run case, which may not correspond with our - // failure - when a fuzz case fails, proptest will try to run at least one more - // case to find a minimal failure case. - *counterexample.borrow_mut() = (calldata, call); - Err(TestCaseError::fail( - decode::decode_revert( - counterexample.borrow().1.result.as_ref(), - errors, - Some(status), - ) - .unwrap_or_default(), - )) } }); @@ -216,6 +213,61 @@ impl<'a> FuzzedExecutor<'a> { result } + + pub fn single_fuzz( + &self, + state: &EvmFuzzState, + address: Address, + should_fail: bool, + calldata: ethers::types::Bytes, + ) -> Result { + let call = self + .executor + .call_raw(self.sender, address, calldata.0.clone(), 0.into()) + .map_err(|_| TestCaseError::fail(FuzzError::FailedContractCall))?; + let state_changeset = call + .state_changeset + .as_ref() + .ok_or_else(|| TestCaseError::fail(FuzzError::EmptyChangeset))?; + + // Build fuzzer state + collect_state_from_call( + &call.logs, + state_changeset, + state.clone(), + &self.config.dictionary, + ); + + // When assume cheat code is triggered return a special string "FOUNDRY::ASSUME" + if call.result.as_ref() == ASSUME_MAGIC_RETURN_CODE { + return Err(TestCaseError::reject(FuzzError::AssumeReject)) + } + + let breakpoints = + call.cheatcodes.clone().map_or(Default::default(), |cheats| cheats.breakpoints); + + let success = + self.executor.is_success(address, call.reverted, state_changeset.clone(), should_fail); + + if success { + Ok(FuzzOutcome::Case(CaseOutcome { + case: FuzzCase { calldata, gas: call.gas_used, stipend: call.stipend }, + gas_used: call.gas_used, + stipend: call.stipend, + traces: call.traces, + coverage: call.coverage, + debug: call.debug, + breakpoints, + })) + } else { + Ok(FuzzOutcome::CounterExample(CounterExampleOutcome { + debug: call.debug.clone(), + exit_reason: call.exit_reason, + counterexample: (calldata, call), + breakpoints, + })) + } + } } #[derive(Clone, Debug, Serialize, Deserialize)] From f470de992a65956e291907435ee2911eddc5f660 Mon Sep 17 00:00:00 2001 From: franfran Date: Tue, 22 Aug 2023 00:39:52 +0300 Subject: [PATCH 2/7] add struct docs --- crates/evm/src/fuzz/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/evm/src/fuzz/mod.rs b/crates/evm/src/fuzz/mod.rs index 572743f6769fc..5ad0a70f48b19 100644 --- a/crates/evm/src/fuzz/mod.rs +++ b/crates/evm/src/fuzz/mod.rs @@ -27,6 +27,8 @@ pub mod error; pub mod invariant; pub mod strategies; +/// Returned by a single fuzz in the case of a successful run +#[derive(Debug)] pub struct CaseOutcome { pub case: FuzzCase, pub gas_used: u64, @@ -37,6 +39,8 @@ pub struct CaseOutcome { pub breakpoints: Breakpoints, } +/// Returned by a single fuzz when a counterexample has been discovered +#[derive(Debug)] pub struct CounterExampleOutcome { pub counterexample: (ethers::types::Bytes, RawCallResult), pub exit_reason: InstructionResult, @@ -44,6 +48,8 @@ pub struct CounterExampleOutcome { pub breakpoints: Breakpoints, } +/// Outcome of a single fuzz +#[derive(Debug)] pub enum FuzzOutcome { Case(CaseOutcome), CounterExample(CounterExampleOutcome), From e016eaa0c421f587c0f9a7614ea1c3af2b83fb1a Mon Sep 17 00:00:00 2001 From: Franfran <51274081+iFrostizz@users.noreply.github.com> Date: Wed, 23 Aug 2023 17:40:14 +0300 Subject: [PATCH 3/7] Update crates/evm/src/fuzz/mod.rs Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- crates/evm/src/fuzz/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/evm/src/fuzz/mod.rs b/crates/evm/src/fuzz/mod.rs index 5ad0a70f48b19..d969a1ac2eda2 100644 --- a/crates/evm/src/fuzz/mod.rs +++ b/crates/evm/src/fuzz/mod.rs @@ -250,7 +250,7 @@ impl<'a> FuzzedExecutor<'a> { } let breakpoints = - call.cheatcodes.clone().map_or(Default::default(), |cheats| cheats.breakpoints); + call.cheatcodes.as_ref().map_or_else(Default::default, |cheats| cheats.breakpoints.clone()); let success = self.executor.is_success(address, call.reverted, state_changeset.clone(), should_fail); From 78dca6a6c59f191aa2a0b737bc78fa2dc1f9b2c2 Mon Sep 17 00:00:00 2001 From: franfran Date: Fri, 25 Aug 2023 11:21:51 +0300 Subject: [PATCH 4/7] add docs and move types to types.rs --- crates/evm/src/fuzz/mod.rs | 47 ++++-------------------------------- crates/evm/src/fuzz/types.rs | 44 +++++++++++++++++++++++++++++++++ crates/forge/src/result.rs | 2 +- 3 files changed, 50 insertions(+), 43 deletions(-) create mode 100644 crates/evm/src/fuzz/types.rs diff --git a/crates/evm/src/fuzz/mod.rs b/crates/evm/src/fuzz/mod.rs index d969a1ac2eda2..9bb357ef8e4e4 100644 --- a/crates/evm/src/fuzz/mod.rs +++ b/crates/evm/src/fuzz/mod.rs @@ -1,6 +1,5 @@ use crate::{ coverage::HitMaps, - debug::DebugArena, decode::{self, decode_console_logs}, executor::{Executor, RawCallResult}, trace::CallTraceArena, @@ -11,49 +10,22 @@ use ethers::{ types::{Address, Bytes, Log}, }; use eyre::Result; -use foundry_common::{calc, contracts::ContractsByAddress, evm::Breakpoints}; +use foundry_common::{calc, contracts::ContractsByAddress}; use foundry_config::FuzzConfig; pub use proptest::test_runner::Reason; use proptest::test_runner::{TestCaseError, TestError, TestRunner}; -use revm::interpreter::InstructionResult; use serde::{Deserialize, Serialize}; use std::{cell::RefCell, collections::BTreeMap, fmt}; use strategies::{ build_initial_state, collect_state_from_call, fuzz_calldata, fuzz_calldata_from_state, EvmFuzzState, }; +use types::{CaseOutcome, CounterExampleOutcome, FuzzCase, FuzzOutcome}; pub mod error; pub mod invariant; pub mod strategies; - -/// Returned by a single fuzz in the case of a successful run -#[derive(Debug)] -pub struct CaseOutcome { - pub case: FuzzCase, - pub gas_used: u64, - pub stipend: u64, - pub traces: Option, - pub coverage: Option, - pub debug: Option, - pub breakpoints: Breakpoints, -} - -/// Returned by a single fuzz when a counterexample has been discovered -#[derive(Debug)] -pub struct CounterExampleOutcome { - pub counterexample: (ethers::types::Bytes, RawCallResult), - pub exit_reason: InstructionResult, - pub debug: Option, - pub breakpoints: Breakpoints, -} - -/// Outcome of a single fuzz -#[derive(Debug)] -pub enum FuzzOutcome { - Case(CaseOutcome), - CounterExample(CounterExampleOutcome), -} +pub mod types; /// Wrapper around an [`Executor`] which provides fuzzing support using [`proptest`](https://docs.rs/proptest/1.0.0/proptest/). /// @@ -220,6 +192,8 @@ impl<'a> FuzzedExecutor<'a> { result } + /// Granular and single-step function that runs only one fuzz and returns either a `CaseOutcome` + /// or a `CounterExampleOutcome` pub fn single_fuzz( &self, state: &EvmFuzzState, @@ -502,14 +476,3 @@ impl FuzzedCases { self.lowest().map(|c| c.gas).unwrap_or_default() } } - -/// Data of a single fuzz test case -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct FuzzCase { - /// The calldata used for this fuzz test - pub calldata: Bytes, - /// Consumed gas - pub gas: u64, - /// The initial gas stipend for the transaction - pub stipend: u64, -} diff --git a/crates/evm/src/fuzz/types.rs b/crates/evm/src/fuzz/types.rs new file mode 100644 index 0000000000000..266bd671d4e84 --- /dev/null +++ b/crates/evm/src/fuzz/types.rs @@ -0,0 +1,44 @@ +use crate::{coverage::HitMaps, debug::DebugArena, executor::RawCallResult, trace::CallTraceArena}; +use ethers::types::Bytes; +use foundry_common::evm::Breakpoints; +use revm::interpreter::InstructionResult; +use serde::{Deserialize, Serialize}; + +/// Data of a single fuzz test case +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct FuzzCase { + /// The calldata used for this fuzz test + pub calldata: Bytes, + /// Consumed gas + pub gas: u64, + /// The initial gas stipend for the transaction + pub stipend: u64, +} + +/// Returned by a single fuzz in the case of a successful run +#[derive(Debug)] +pub struct CaseOutcome { + pub case: FuzzCase, + pub gas_used: u64, + pub stipend: u64, + pub traces: Option, + pub coverage: Option, + pub debug: Option, + pub breakpoints: Breakpoints, +} + +/// Returned by a single fuzz when a counterexample has been discovered +#[derive(Debug)] +pub struct CounterExampleOutcome { + pub counterexample: (ethers::types::Bytes, RawCallResult), + pub exit_reason: InstructionResult, + pub debug: Option, + pub breakpoints: Breakpoints, +} + +/// Outcome of a single fuzz +#[derive(Debug)] +pub enum FuzzOutcome { + Case(CaseOutcome), + CounterExample(CounterExampleOutcome), +} diff --git a/crates/forge/src/result.rs b/crates/forge/src/result.rs index c6363e0695ce2..ca717e61b6484 100644 --- a/crates/forge/src/result.rs +++ b/crates/forge/src/result.rs @@ -6,7 +6,7 @@ use foundry_common::evm::Breakpoints; use foundry_evm::{ coverage::HitMaps, executor::EvmError, - fuzz::{CounterExample, FuzzCase}, + fuzz::{types::FuzzCase, CounterExample}, trace::{TraceKind, Traces}, }; use serde::{Deserialize, Serialize}; From 15e4709a78aa935c2607a35aad7955c28ca70fce Mon Sep 17 00:00:00 2001 From: franfran Date: Fri, 25 Aug 2023 11:24:19 +0300 Subject: [PATCH 5/7] fmt --- crates/evm/src/fuzz/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/evm/src/fuzz/mod.rs b/crates/evm/src/fuzz/mod.rs index 9bb357ef8e4e4..4bdd997921d6f 100644 --- a/crates/evm/src/fuzz/mod.rs +++ b/crates/evm/src/fuzz/mod.rs @@ -223,8 +223,10 @@ impl<'a> FuzzedExecutor<'a> { return Err(TestCaseError::reject(FuzzError::AssumeReject)) } - let breakpoints = - call.cheatcodes.as_ref().map_or_else(Default::default, |cheats| cheats.breakpoints.clone()); + let breakpoints = call + .cheatcodes + .as_ref() + .map_or_else(Default::default, |cheats| cheats.breakpoints.clone()); let success = self.executor.is_success(address, call.reverted, state_changeset.clone(), should_fail); From 9c0011c53acb4021f1ff2b8218292eec97dd6107 Mon Sep 17 00:00:00 2001 From: franfran Date: Sun, 27 Aug 2023 18:45:24 +0300 Subject: [PATCH 6/7] add docki docs --- crates/evm/src/fuzz/mod.rs | 1 - crates/evm/src/fuzz/types.rs | 11 +++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/evm/src/fuzz/mod.rs b/crates/evm/src/fuzz/mod.rs index 4bdd997921d6f..52f037705c857 100644 --- a/crates/evm/src/fuzz/mod.rs +++ b/crates/evm/src/fuzz/mod.rs @@ -234,7 +234,6 @@ impl<'a> FuzzedExecutor<'a> { if success { Ok(FuzzOutcome::Case(CaseOutcome { case: FuzzCase { calldata, gas: call.gas_used, stipend: call.stipend }, - gas_used: call.gas_used, stipend: call.stipend, traces: call.traces, coverage: call.coverage, diff --git a/crates/evm/src/fuzz/types.rs b/crates/evm/src/fuzz/types.rs index 266bd671d4e84..7178376a67a74 100644 --- a/crates/evm/src/fuzz/types.rs +++ b/crates/evm/src/fuzz/types.rs @@ -18,21 +18,28 @@ pub struct FuzzCase { /// Returned by a single fuzz in the case of a successful run #[derive(Debug)] pub struct CaseOutcome { + /// Data of a single fuzz test case pub case: FuzzCase, - pub gas_used: u64, - pub stipend: u64, + /// The traces of the call pub traces: Option, + /// The coverage info collected during the call pub coverage: Option, + /// The debug nodes of the call pub debug: Option, + /// Breakpoints char pc map pub breakpoints: Breakpoints, } /// Returned by a single fuzz when a counterexample has been discovered #[derive(Debug)] pub struct CounterExampleOutcome { + /// Minimal reproduction test case for failing test pub counterexample: (ethers::types::Bytes, RawCallResult), + /// The status of the call pub exit_reason: InstructionResult, + /// The debug nodes of the call pub debug: Option, + /// Breakpoints char pc map pub breakpoints: Breakpoints, } From e811123bb4d8db243b77174357e2b4c94085451f Mon Sep 17 00:00:00 2001 From: franfran Date: Sun, 27 Aug 2023 19:01:55 +0300 Subject: [PATCH 7/7] fmt --- crates/evm/src/fuzz/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/evm/src/fuzz/mod.rs b/crates/evm/src/fuzz/mod.rs index 52f037705c857..e0dc9e466fbba 100644 --- a/crates/evm/src/fuzz/mod.rs +++ b/crates/evm/src/fuzz/mod.rs @@ -108,10 +108,10 @@ impl<'a> FuzzedExecutor<'a> { match fuzz_res { FuzzOutcome::Case(case) => { let mut first_case = first_case.borrow_mut(); + gas_by_case.borrow_mut().push((case.case.gas, case.case.stipend)); if first_case.is_none() { first_case.replace(case.case); } - gas_by_case.borrow_mut().push((case.gas_used, case.stipend)); traces.replace(case.traces); @@ -234,7 +234,6 @@ impl<'a> FuzzedExecutor<'a> { if success { Ok(FuzzOutcome::Case(CaseOutcome { case: FuzzCase { calldata, gas: call.gas_used, stipend: call.stipend }, - stipend: call.stipend, traces: call.traces, coverage: call.coverage, debug: call.debug,