diff --git a/Cargo.lock b/Cargo.lock index 48f6e8efe1791..6a3431702a517 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -384,7 +384,7 @@ dependencies = [ "paste", "proptest", "proptest-derive", - "rand 0.9.1", + "rand 0.9.2", "ruint", "rustc-hash 2.1.1", "serde", @@ -1083,7 +1083,7 @@ dependencies = [ "op-revm", "parking_lot", "rand 0.8.5", - "rand 0.9.1", + "rand 0.9.2", "revm", "revm-inspectors", "serde", @@ -1113,7 +1113,7 @@ dependencies = [ "foundry-evm", "op-alloy-consensus 0.17.2", "op-revm", - "rand 0.9.1", + "rand 0.9.2", "revm", "serde", "serde_json", @@ -2420,7 +2420,7 @@ dependencies = [ "op-alloy-consensus 0.17.2", "op-alloy-flz", "rand 0.8.5", - "rand 0.9.1", + "rand 0.9.2", "rayon", "regex", "revm", @@ -3880,21 +3880,25 @@ dependencies = [ "inferno", "itertools 0.14.0", "mockall", + "num-bigint", "opener 0.7.2", "parking_lot", "paste", "path-slash", "proptest", "quick-junit", + "rand 0.9.2", "rayon", "regex", "reqwest", "revm", + "rstest", "semver 1.0.26", "serde", "serde_json", "similar", "similar-asserts", + "solar-interface", "solar-parse", "solar-sema", "soldeer-commands", @@ -4165,7 +4169,7 @@ dependencies = [ "p256", "parking_lot", "proptest", - "rand 0.9.1", + "rand 0.9.2", "revm", "revm-inspectors", "semver 1.0.26", @@ -4591,7 +4595,7 @@ dependencies = [ "itertools 0.14.0", "parking_lot", "proptest", - "rand 0.9.1", + "rand 0.9.2", "revm", "serde", "thiserror 2.0.12", @@ -4698,7 +4702,7 @@ dependencies = [ "foundry-config", "idna_adapter", "parking_lot", - "rand 0.9.1", + "rand 0.9.2", "regex", "serde_json", "snapbox", @@ -4859,6 +4863,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.31" @@ -7230,7 +7240,7 @@ dependencies = [ "bitflags 2.9.1", "lazy_static", "num-traits", - "rand 0.9.1", + "rand 0.9.2", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax 0.8.5", @@ -7388,7 +7398,7 @@ dependencies = [ "bytes", "getrandom 0.3.3", "lru-slab", - "rand 0.9.1", + "rand 0.9.2", "ring", "rustc-hash 2.1.1", "rustls", @@ -7459,9 +7469,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", @@ -7565,9 +7575,9 @@ checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" [[package]] name = "redox_syscall" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" +checksum = "de3a5d9f0aba1dbcec1cc47f0ff94a4b778fe55bca98a6dfa92e4e094e57b1c4" dependencies = [ "bitflags 2.9.1", ] @@ -7653,6 +7663,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.22" @@ -7972,6 +7988,36 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rstest" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", + "rustc_version 0.4.1", +] + +[[package]] +name = "rstest_macros" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version 0.4.1", + "syn 2.0.104", + "unicode-ident", +] + [[package]] name = "rtoolbox" version = "0.0.3" @@ -8014,7 +8060,7 @@ dependencies = [ "primitive-types", "proptest", "rand 0.8.5", - "rand 0.9.1", + "rand 0.9.2", "rlp", "ruint-macro", "serde", @@ -8359,7 +8405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c3c81b43dc2d8877c216a3fccf76677ee1ebccd429566d3e67447290d0c42b2" dependencies = [ "bitcoin_hashes", - "rand 0.9.1", + "rand 0.9.2", "secp256k1-sys 0.11.0", ] @@ -9875,7 +9921,7 @@ dependencies = [ "http 1.3.1", "httparse", "log", - "rand 0.9.1", + "rand 0.9.2", "rustls", "rustls-pki-types", "sha1", diff --git a/cache/solidity-files-cache.json b/cache/solidity-files-cache.json new file mode 100644 index 0000000000000..62081cb1cc34f --- /dev/null +++ b/cache/solidity-files-cache.json @@ -0,0 +1 @@ +{"_format":"","paths":{"artifacts":"out","build_infos":"out/build-info","sources":"src","tests":"test","scripts":"script","libraries":["lib"]},"files":{"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_BitNot/src/Counter.sol":{"lastModificationDate":1743435364548,"contentHash":"409e7881b1f12b7eda886da0aadf38bd","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_BitNot/src/Counter.sol","imports":[],"versionRequirement":"^0.8.13","artifacts":{"Counter":{"0.8.28":{"default":{"path":"mutation_211_203_UnaryOperator_BitNot/src/Counter.sol/Counter.json","build_id":"611b06cf8f63cf633d8f3e1cf644bbfe"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_BitNot/test/CounterTest.t.sol":{"lastModificationDate":1743435364534,"contentHash":"7ca0c40d195ed5b4cdc882be42cba837","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_BitNot/test/CounterTest.t.sol","imports":["/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_BitNot/src/Counter.sol"],"versionRequirement":"^0.8.13","artifacts":{"CounterTest":{"0.8.28":{"default":{"path":"mutation_211_203_UnaryOperator_BitNot/test/CounterTest.t.sol/CounterTest.json","build_id":"611b06cf8f63cf633d8f3e1cf644bbfe"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PostDec/src/Counter.sol":{"lastModificationDate":1743435364548,"contentHash":"23ec06146f4e636071d9d2f687bb39b8","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PostDec/src/Counter.sol","imports":[],"versionRequirement":"^0.8.13","artifacts":{"Counter":{"0.8.28":{"default":{"path":"mutation_211_203_UnaryOperator_PostDec/src/Counter.sol/Counter.json","build_id":"b733849b9e35a95f8a64b14dda9151ef"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PostDec/test/CounterTest.t.sol":{"lastModificationDate":1743435364534,"contentHash":"7ca0c40d195ed5b4cdc882be42cba837","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PostDec/test/CounterTest.t.sol","imports":["/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PostDec/src/Counter.sol"],"versionRequirement":"^0.8.13","artifacts":{"CounterTest":{"0.8.28":{"default":{"path":"mutation_211_203_UnaryOperator_PostDec/test/CounterTest.t.sol/CounterTest.json","build_id":"b733849b9e35a95f8a64b14dda9151ef"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreDec/src/Counter.sol":{"lastModificationDate":1743435364548,"contentHash":"fa61ead0fe3f3d3cf9a74043ca8a1cca","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreDec/src/Counter.sol","imports":[],"versionRequirement":"^0.8.13","artifacts":{"Counter":{"0.8.28":{"default":{"path":"src/Counter.sol/Counter.json","build_id":"2068d3446326d754b730a2df5ea568f8"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreDec/test/CounterTest.t.sol":{"lastModificationDate":1743435364534,"contentHash":"7ca0c40d195ed5b4cdc882be42cba837","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreDec/test/CounterTest.t.sol","imports":["/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreDec/src/Counter.sol"],"versionRequirement":"^0.8.13","artifacts":{"CounterTest":{"0.8.28":{"default":{"path":"test/CounterTest.t.sol/CounterTest.json","build_id":"2068d3446326d754b730a2df5ea568f8"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreInc/src/Counter.sol":{"lastModificationDate":1743435364548,"contentHash":"1129c6e517e1881612a094060e2866bd","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreInc/src/Counter.sol","imports":[],"versionRequirement":"^0.8.13","artifacts":{"Counter":{"0.8.28":{"default":{"path":"Counter.sol/Counter.json","build_id":"32fa3bbd0fbf144bf1fd0af3ea20e90e"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreInc/test/CounterTest.t.sol":{"lastModificationDate":1743435364534,"contentHash":"7ca0c40d195ed5b4cdc882be42cba837","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreInc/test/CounterTest.t.sol","imports":["/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreInc/src/Counter.sol"],"versionRequirement":"^0.8.13","artifacts":{"CounterTest":{"0.8.28":{"default":{"path":"CounterTest.t.sol/CounterTest.json","build_id":"32fa3bbd0fbf144bf1fd0af3ea20e90e"}}}},"seenByCompiler":true}},"builds":["2068d3446326d754b730a2df5ea568f8","32fa3bbd0fbf144bf1fd0af3ea20e90e","611b06cf8f63cf633d8f3e1cf644bbfe","b733849b9e35a95f8a64b14dda9151ef"],"profiles":{"default":{"solc":{"optimizer":{"enabled":false,"runs":200},"metadata":{"useLiteralContent":false,"bytecodeHash":"ipfs","appendCBOR":true},"outputSelection":{"*":{"*":["abi","evm.bytecode.object","evm.bytecode.sourceMap","evm.bytecode.linkReferences","evm.deployedBytecode.object","evm.deployedBytecode.sourceMap","evm.deployedBytecode.linkReferences","evm.deployedBytecode.immutableReferences","evm.methodIdentifiers","metadata"]}},"evmVersion":"cancun","viaIR":false,"libraries":{}},"vyper":{"evmVersion":"cancun","outputSelection":{"*":{"*":["abi","evm.bytecode","evm.deployedBytecode"]}}}}}} \ No newline at end of file diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 9d265867a11d3..85e462a065c4e 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -323,6 +323,8 @@ pub struct Config { pub coverage_pattern_inverse: Option, /// Path where last test run failures are recorded. pub test_failures_file: PathBuf, + /// Pathe where mutation tests are cached, to resume running them + pub mutation_dir: PathBuf, /// Max concurrent threads to use. pub threads: Option, /// Whether to show test execution progress. @@ -1060,6 +1062,9 @@ impl Config { // Remove last test run failures file. let _ = fs::remove_file(&self.test_failures_file); + // Remove mutation test cache directory + let _ = fs::remove_dir_all(project.root().join(&self.mutation_dir)); + // Remove fuzz and invariant cache directories. let remove_test_dir = |test_dir: &Option| { if let Some(test_dir) = test_dir { @@ -2367,6 +2372,7 @@ impl Default for Config { path_pattern_inverse: None, coverage_pattern_inverse: None, test_failures_file: "cache/test-failures".into(), + mutation_dir: "cache/mutation".into(), threads: None, show_progress: false, fuzz: FuzzConfig::new("cache/fuzz".into()), diff --git a/crates/forge/Cargo.toml b/crates/forge/Cargo.toml index ee5727837862c..1d080b418d801 100644 --- a/crates/forge/Cargo.toml +++ b/crates/forge/Cargo.toml @@ -62,8 +62,6 @@ alloy-serde.workspace = true alloy-signer.workspace = true alloy-transport.workspace = true -revm.workspace = true - clap = { version = "4", features = ["derive", "env", "unicode", "wrap_help"] } clap_complete = "4" clap_complete_fig = "4" @@ -72,14 +70,19 @@ indicatif.workspace = true inferno = { version = "0.12", default-features = false } itertools.workspace = true parking_lot.workspace = true +rand.workspace = true regex = { workspace = true, default-features = false } +reqwest = { workspace = true, features = ["json"] } +revm.workspace = true semver.workspace = true serde_json.workspace = true similar = { version = "2", features = ["inline"] } solang-parser.workspace = true +solar-interface.workspace = true solar-parse.workspace = true solar-sema.workspace = true strum = { workspace = true, features = ["derive"] } +tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["time"] } toml_edit = "0.22" @@ -88,6 +91,7 @@ watchexec-events = "6.0" watchexec-signals = "5.0" clearscreen = "4.0" evm-disassembler.workspace = true +num-bigint = "0.4" path-slash.workspace = true # doc server @@ -111,6 +115,8 @@ reqwest = { workspace = true, features = ["json"] } mockall = "0.13" globset = "0.4" paste = "1.0" + +rstest = "0.25.0" similar-asserts.workspace = true svm = { package = "svm-rs", version = "0.5", default-features = false, features = [ "rustls", diff --git a/crates/forge/src/cmd/test/mod.rs b/crates/forge/src/cmd/test/mod.rs index e6171f0cf17c5..847528e5dc51e 100644 --- a/crates/forge/src/cmd/test/mod.rs +++ b/crates/forge/src/cmd/test/mod.rs @@ -4,6 +4,7 @@ use crate::{ decode::decode_console_logs, gas_report::GasReport, multi_runner::matches_contract, + mutation::{MutationHandler, MutationReporter, MutationsSummary}, result::{SuiteResult, TestOutcome, TestStatus}, traces::{ CallTraceDecoderBuilder, InternalTraceMode, TraceKind, @@ -19,7 +20,7 @@ use eyre::{Context, OptionExt, Result, bail}; use foundry_block_explorers::EtherscanApiVersion; use foundry_cli::{ opts::{BuildOpts, GlobalArgs}, - utils::{self, LoadConfig}, + utils::{self, FoundryPathExt, LoadConfig}, }; use foundry_common::{TestFunctionExt, compile::ProjectCompiler, evm::EvmArgs, fs, shell}; use foundry_compilers::{ @@ -197,6 +198,19 @@ pub struct TestArgs { #[command(flatten)] pub watch: WatchArgs, + + /// Enable mutation testing. + /// If passed with file paths, only those files will be tested. + #[arg(long, num_args(0..), value_name = "PATH")] + pub mutate: Option>, + + /// Specify which files to mutate with glob pattern matching. + #[arg(long, value_name = "PATTERN", requires = "mutate")] + pub mutate_path: Option, + + /// Only run tests in contracts matching the specified regex pattern. + #[arg(long, value_name = "REGEX", requires = "mutate")] + pub mutate_contract: Option, } impl TestArgs { @@ -295,6 +309,14 @@ impl TestArgs { config.invariant.gas_report_samples = 0; } + let should_mutate = self.mutate.is_some(); + + // Force dyn test linking for mutation testing + if should_mutate { + config.dynamic_test_linking = true; + config.cache = true; + } + // Install missing dependencies. if install::install_missing_dependencies(&mut config) && config.auto_detect_remappings { // need to re-configure here to also catch additional remappings @@ -355,10 +377,11 @@ impl TestArgs { .with_fork(evm_opts.get_fork(&config, env.clone())) .enable_isolation(evm_opts.isolate) .odyssey(evm_opts.odyssey) - .build::(project_root, &output, env, evm_opts)?; + .build::(project_root, &output, env.clone(), evm_opts.clone())?; let libraries = runner.libraries.clone(); - let mut outcome = self.run_tests(runner, config, verbosity, &filter, &output).await?; + let mut outcome = + self.run_tests(runner, config.clone(), verbosity, &filter, &output).await?; if should_draw { let (suite_name, test_name, mut test_result) = @@ -422,13 +445,171 @@ impl TestArgs { } let mut debugger = builder.build(); - if let Some(dump_path) = self.dump { + if let Some(dump_path) = self.dump.clone() { debugger.dump_to_file(&dump_path)?; } else { debugger.try_run_tui()?; } } + // All test have been run once before reaching this point + if should_mutate { + // check outcome here, stop if any test failed + // @todo rather set non-allowed failed tests in config and ensure_ok() here? + // @todo other checks: no fork (or just exclude based on clap arg?) + if outcome.failed() > 0 { + eyre::bail!("Cannot run mutation testing with failed tests"); + } + + let mutate_paths = if let Some(pattern) = &self.mutate_path { + // If --mutate-path is provided, use it to filter paths + source_files_iter(&project.paths.sources, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| { + // @todo filter out interfaces here? + // we do it in lexing for now + entry.is_sol() && !entry.is_sol_test() && pattern.is_match(entry) + }) + .collect() + } else if let Some(contract_pattern) = &self.mutate_contract { + // If --mutate-contract is provided, use it to filter contracts + source_files_iter(&project.paths.sources, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| { + entry.is_sol() + && !entry.is_sol_test() + && output + .artifact_ids() + .find(|(id, _)| id.source == *entry) + .is_some_and(|(id, _)| contract_pattern.is_match(&id.name)) + }) + .collect() + } else if self.mutate.as_ref().unwrap().is_empty() { + // If --mutate is passed without arguments, use all Solidity files + source_files_iter(&project.paths.sources, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| entry.is_sol() && !entry.is_sol_test()) + .collect() + } else { + // If --mutate is passed with arguments, use those paths + self.mutate.as_ref().unwrap().clone() + }; + + sh_println!("Running mutation tests...").unwrap(); + let mut mutation_summary = MutationsSummary::new(); + + for path in mutate_paths { + sh_println!("Running mutation tests for {}", path.display()).unwrap(); + + // Check if this file has already been tested and if the build id is the + // same - if so, just add the mutants to the summary + let mut handler = MutationHandler::new(path.clone(), config.clone()); + + handler.read_source_contract()?; + + let build_id = output + .artifact_ids() + .find_map( + |(id, _)| if id.source == path { Some(id.build_id.clone()) } else { None }, + ) + .unwrap_or_default(); + + // If we have cached results for these mutations and build id, use them and skip + // running tests + if let Some(prior) = handler.retrieve_cached_mutant_results(&build_id) { + for (mutant, status) in prior { + match status { + crate::mutation::mutant::MutationResult::Dead => { + mutation_summary.add_dead_mutant(mutant) + } + crate::mutation::mutant::MutationResult::Alive => { + mutation_summary.add_survived_mutant(mutant) + } + crate::mutation::mutant::MutationResult::Invalid => { + mutation_summary.update_invalid_mutant(mutant) + } + } + } + continue; + } + + // Try cached mutants first + let mut mutants = if let Some(ms) = handler.retrieve_cached_mutants(&build_id) { + ms + } else { + // No cache match: generate fresh mutants + handler.generate_ast().await; + handler.mutations.clone() + }; + + // Accumulate per-mutant results for persistence + let mut results_vec: Vec<( + crate::mutation::mutant::Mutant, + crate::mutation::mutant::MutationResult, + )> = Vec::with_capacity(mutants.len()); + + for (i, mutant) in mutants.iter().enumerate() { + sh_println!("Testing mutant {} out of {}", i + 1, mutants.len()).unwrap(); + + handler.generate_mutated_solidity(&mutant); + let new_filter = self.filter(&config).unwrap(); + let compiler = ProjectCompiler::new() + .dynamic_test_linking(config.dynamic_test_linking) + .quiet(true); + + let compile_output = compiler.compile(&project); + + if compile_output.is_err() { + mutation_summary.update_invalid_mutant(mutant.clone()); + results_vec.push(( + mutant.clone(), + crate::mutation::mutant::MutationResult::Invalid, + )); + } else { + let mut runner = MultiContractRunnerBuilder::new(config.clone()) + .set_debug(false) + .initial_balance(evm_opts.initial_balance) + .evm_spec(config.evm_spec_id()) + .sender(evm_opts.sender) + .odyssey(evm_opts.odyssey) + .build::( + &config.root, + &compile_output.unwrap(), + env.clone(), + evm_opts.clone(), + )?; + + let results: BTreeMap = + runner.test_collect(&new_filter)?; + + let outcome = TestOutcome::new(results, self.allow_failure); + if outcome.failures().count() > 0 { + mutation_summary.add_dead_mutant(mutant.clone()); + results_vec.push(( + mutant.clone(), + crate::mutation::mutant::MutationResult::Dead, + )); + } else { + mutation_summary.add_survived_mutant(mutant.clone()); + results_vec.push(( + mutant.clone(), + crate::mutation::mutant::MutationResult::Alive, + )); + } + } + } + + handler.restore_original_source(); + + // If we generated fresh mutants, persist them for this build id + if handler.mutations.len() > 0 && !build_id.is_empty() { + let _ = handler.persist_cached_mutants(&build_id, &handler.mutations); + let _ = handler.persist_cached_results(&build_id, &results_vec); + } + } + + MutationReporter::new().report(&mutation_summary); + + outcome = TestOutcome::empty(true); + } + Ok(outcome) } diff --git a/crates/forge/src/lib.rs b/crates/forge/src/lib.rs index fdbad8e4e6b2e..218cdff813551 100644 --- a/crates/forge/src/lib.rs +++ b/crates/forge/src/lib.rs @@ -20,6 +20,8 @@ pub mod gas_report; pub mod multi_runner; pub use multi_runner::{MultiContractRunner, MultiContractRunnerBuilder}; +pub mod mutation; + mod runner; pub use runner::ContractRunner; diff --git a/crates/forge/src/mutation/mod.rs b/crates/forge/src/mutation/mod.rs new file mode 100644 index 0000000000000..1b768e8c28932 --- /dev/null +++ b/crates/forge/src/mutation/mod.rs @@ -0,0 +1,593 @@ +pub mod mutant; +mod mutators; +mod reporter; +mod visitor; + +// Generate mutants then run tests (reuse the whole unit test flow for now, including compilation to +// select mutants) Use Solar: +use solar_parse::{ + Parser, + ast::interface::{Session, source_map::FileName}, +}; +use std::sync::Arc; + +use crate::mutation::{mutant::Mutant, visitor::MutantVisitor}; + +pub use crate::mutation::reporter::MutationReporter; + +use crate::result::TestOutcome; +use dunce; +use foundry_common::fs; +use foundry_compilers::{ + ProjectCompileOutput, cache::SOLIDITY_FILES_CACHE_FILENAME, project::ProjectCompiler, +}; +use foundry_config::Config; +use rayon::prelude::*; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use serde_json; +use solar_interface::BytePos; +use solar_parse::ast::visit::Visit; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +pub struct MutationsSummary { + total: usize, + dead: Vec, + survived: Vec, + invalid: Vec, +} + +impl MutationsSummary { + pub fn new() -> Self { + Self { total: 0, dead: vec![], survived: vec![], invalid: vec![] } + } + + pub fn update_valid_mutant(&mut self, outcome: &TestOutcome, mutant: Mutant) { + if outcome.failures().count() > 0 { + self.dead.push(mutant); + } else { + self.survived.push(mutant); + } + } + + pub fn update_invalid_mutant(&mut self, mutant: Mutant) { + self.invalid.push(mutant); + } + + pub fn add_dead_mutant(&mut self, mutant: Mutant) { + self.dead.push(mutant); + } + + pub fn add_survived_mutant(&mut self, mutant: Mutant) { + self.survived.push(mutant); + } + + pub fn total_mutants(&self) -> usize { + self.dead.len() + self.survived.len() + self.invalid.len() + } + + pub fn total_dead(&self) -> usize { + self.dead.len() + } + + pub fn total_survived(&self) -> usize { + self.survived.len() + } + + pub fn total_invalid(&self) -> usize { + self.invalid.len() + } + + pub fn dead(&self) -> String { + self.dead.iter().map(|m| m.to_string()).collect::>().join("\n") + } + + pub fn survived(&self) -> String { + self.survived.iter().map(|m| m.to_string()).collect::>().join("\n") + } + + pub fn invalid(&self) -> String { + self.invalid.iter().map(|m| m.to_string()).collect::>().join("\n") + } +} + +pub struct MutationHandler { + contract_to_mutate: PathBuf, + hash_build: String, + src: Arc, + pub mutations: Vec, + config: Arc, + report: MutationsSummary, +} + +impl MutationHandler { + pub fn new(contract_to_mutate: PathBuf, config: Arc) -> Self { + Self { + contract_to_mutate, + hash_build: String::new(), + src: Arc::default(), + mutations: vec![], + config, + report: MutationsSummary::new(), + } + } + + pub fn read_source_contract(&mut self) -> Result<(), std::io::Error> { + let content = std::fs::read_to_string(&self.contract_to_mutate)?; + self.src = Arc::new(content); + Ok(()) + } + + // Note: we now get the build hash directly from the recent compile output (see test flow) + + /// Persists the mapping entry for this contract and writes the cached mutants JSON file + /// at `cache/mutation/.mutants`. + pub fn persist_cached_mutants(&self, hash: &str, mutants: &[Mutant]) -> std::io::Result<()> { + #[derive(Serialize, Deserialize)] + #[serde(tag = "kind")] + enum MutationDtoKind { + AssignmentLiteral { lit: String }, + AssignmentIdentifier { ident: String }, + BinaryOp { op: String }, + DeleteExpression, + ElimDelegate, + FunctionCall, + Require, + SwapArgumentsFunction, + SwapArgumentsOperator, + UnaryOperator { expr: String, op: String }, + } + + #[derive(Serialize, Deserialize)] + struct MutantDto { + path: String, + lo: u64, + hi: u64, + mutation: MutationDtoKind, + } + + let mutation_cache_dir = self.config.root.join(&self.config.mutation_dir); + std::fs::create_dir_all(&mutation_cache_dir)?; + + // Update mapping.json with absolute path -> build hash + let mapping_path = mutation_cache_dir.join("mapping.json"); + let mut mapping: HashMap = std::fs::read_to_string(&mapping_path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + + let contract_path = dunce::canonicalize(&self.contract_to_mutate) + .unwrap_or_else(|_| self.contract_to_mutate.clone()) + .to_string_lossy() + .into_owned(); + mapping.insert(contract_path, hash.to_string()); + + let mapping_json = serde_json::to_string_pretty(&mapping) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + std::fs::write(&mapping_path, mapping_json)?; + + // Write .mutants with a simple JSON array + let dtos: Vec = mutants + .iter() + .map(|m| { + let mutation = match &m.mutation { + crate::mutation::mutant::MutationType::Assignment(assign) => match assign { + crate::mutation::visitor::AssignVarTypes::Literal(lit) => { + MutationDtoKind::AssignmentLiteral { + lit: lit.description().to_string(), + } + } + crate::mutation::visitor::AssignVarTypes::Identifier(ident) => { + MutationDtoKind::AssignmentIdentifier { ident: ident.clone() } + } + }, + crate::mutation::mutant::MutationType::BinaryOp(kind) => { + MutationDtoKind::BinaryOp { op: kind.to_str().to_string() } + } + crate::mutation::mutant::MutationType::DeleteExpression => { + MutationDtoKind::DeleteExpression + } + crate::mutation::mutant::MutationType::ElimDelegate => { + MutationDtoKind::ElimDelegate + } + crate::mutation::mutant::MutationType::FunctionCall => { + MutationDtoKind::FunctionCall + } + crate::mutation::mutant::MutationType::Require => MutationDtoKind::Require, + crate::mutation::mutant::MutationType::SwapArgumentsFunction => { + MutationDtoKind::SwapArgumentsFunction + } + crate::mutation::mutant::MutationType::SwapArgumentsOperator => { + MutationDtoKind::SwapArgumentsOperator + } + crate::mutation::mutant::MutationType::UnaryOperator(u) => { + MutationDtoKind::UnaryOperator { + expr: u.to_string(), + op: format!("{:?}", u.resulting_op_kind), + } + } + }; + + MutantDto { + path: m.path.to_string_lossy().into_owned(), + lo: m.span.lo().0 as u64, + hi: m.span.hi().0 as u64, + mutation, + } + }) + .collect(); + + let contract_name = self + .contract_to_mutate + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("contract") + .to_string(); + let mutants_file = mutation_cache_dir.join(format!("{contract_name}.mutants")); + let json = serde_json::to_string_pretty(&dtos) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + std::fs::write(mutants_file, json)?; + + Ok(()) + } + + /// Persists results for mutants for given build hash at `cache/mutation/.results`. + pub fn persist_cached_results( + &self, + hash: &str, + results: &[(Mutant, crate::mutation::mutant::MutationResult)], + ) -> std::io::Result<()> { + #[derive(Serialize)] + struct ResultDto { + path: String, + lo: u64, + hi: u64, + status: String, + } + + let mutation_cache_dir = self.config.root.join(&self.config.mutation_dir); + std::fs::create_dir_all(&mutation_cache_dir)?; + + let serialized: Vec = results + .iter() + .map(|(m, r)| ResultDto { + path: m.path.to_string_lossy().into_owned(), + lo: m.span.lo().0 as u64, + hi: m.span.hi().0 as u64, + status: match r { + crate::mutation::mutant::MutationResult::Dead => "dead".to_string(), + crate::mutation::mutant::MutationResult::Alive => "alive".to_string(), + crate::mutation::mutant::MutationResult::Invalid => "invalid".to_string(), + }, + }) + .collect(); + + let contract_name = self + .contract_to_mutate + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("contract") + .to_string(); + let results_file = mutation_cache_dir.join(format!("{contract_name}.results")); + let json = serde_json::to_string_pretty(&serialized) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + std::fs::write(results_file, json)?; + + Ok(()) + } + + /// Read a source string, and for each contract found, gets its ast and visit it to list + /// all mutations to conduct + pub async fn generate_ast(&mut self) { + let path = &self.contract_to_mutate; + let target_content = Arc::clone(&self.src); + let sess = Session::builder().with_silent_emitter(None).build(); + + let _ = sess.enter(|| -> solar_parse::interface::Result<()> { + let arena = solar_parse::ast::Arena::new(); + let mut parser = + Parser::from_lazy_source_code(&sess, &arena, FileName::from(path.clone()), || { + Ok((*target_content).to_string()) + })?; + + let ast = parser.parse_file().map_err(|e| e.emit())?; + + let mut mutant_visitor = MutantVisitor::default(path.clone()); + mutant_visitor.visit_source_unit(&ast); + self.mutations.extend(mutant_visitor.mutation_to_conduct); + Ok(()) + }); + } + + /// Based on a given mutation, emit the corresponding mutated solidity code and write it to disk + pub fn generate_mutated_solidity(&self, mutation: &Mutant) { + let span = mutation.span; + let replacement = mutation.mutation.to_string(); + + let src_content = Arc::clone(&self.src); + + let start_pos = span.lo().0 as usize; + let end_pos = span.hi().0 as usize; + + let before = &src_content[..start_pos]; + let after = &src_content[end_pos..]; + + let mut new_content = String::with_capacity(before.len() + replacement.len() + after.len()); + new_content.push_str(before); + new_content.push_str(&replacement); + new_content.push_str(after); + + std::fs::write(&self.contract_to_mutate, new_content).unwrap_or_else(|_| { + panic!("Failed to write to target file {:?}", &self.contract_to_mutate) + }); + } + + // @todo src to mutate should be in a tmp dir for safety (and modify config accordingly) + /// Restore the original source contract to the target file (end of mutation tests) + pub fn restore_original_source(&self) { + std::fs::write(&self.contract_to_mutate, &*self.src).unwrap_or_else(|_| { + panic!("Failed to write to target file {:?}", &self.contract_to_mutate) + }); + } + + // get the file which hold a mapping `contract to mutate`->hash build + // - if target contract doesn't exist in it, return None + // - if target contract exist, get the hash build: + // -- if hash build is the same as the one passed as argument, load the mutants from the + // hash.mutants file and return Some(mutants) + // -- if hash build is different, remove it from the mapping file and return None + pub fn retrieve_cached_mutants(&self, hash: &str) -> Option> { + // mutation cache directory under the project root + let mutation_cache_dir = self.config.root.join(&self.config.mutation_dir); + let mapping_path = mutation_cache_dir.join("mapping.json"); + + // Read mapping file `{contract_absolute_path -> build_hash}` + let mapping: HashMap = match std::fs::read_to_string(&mapping_path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + { + Some(map) => map, + None => return None, + }; + + // Canonicalize the contract path to match mapping keys + let contract_path = dunce::canonicalize(&self.contract_to_mutate) + .unwrap_or_else(|_| self.contract_to_mutate.clone()) + .to_string_lossy() + .into_owned(); + + if let Some(stored_hash) = mapping.get(&contract_path) { + if stored_hash == hash { + // Try to read the cached mutants file for this build hash + let contract_name = self + .contract_to_mutate + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("contract") + .to_string(); + let mutants_file = mutation_cache_dir.join(format!("{contract_name}.mutants")); + if mutants_file.exists() { + if let Ok(data) = std::fs::read_to_string(&mutants_file) { + #[derive(Deserialize)] + #[serde(tag = "kind")] + enum MutationDtoKindRead { + AssignmentLiteral { lit: String }, + AssignmentIdentifier { ident: String }, + BinaryOp { op: String }, + DeleteExpression, + ElimDelegate, + FunctionCall, + Require, + SwapArgumentsFunction, + SwapArgumentsOperator, + UnaryOperator { expr: String, op: String }, + } + #[derive(Deserialize)] + struct MutantDtoRead { + path: String, + lo: u64, + hi: u64, + mutation: MutationDtoKindRead, + } + + if let Ok(raw_mutants) = serde_json::from_str::>(&data) { + let mut out: Vec = Vec::new(); + for m in raw_mutants { + let span = solar_parse::ast::Span::new( + BytePos(m.lo as u32), + BytePos(m.hi as u32), + ); + let mutation = match m.mutation { + MutationDtoKindRead::AssignmentLiteral { lit } => { + let lit_kind = match lit.as_str() { + "true" => solar_parse::ast::LitKind::Bool(true), + "false" => solar_parse::ast::LitKind::Bool(false), + _ => solar_parse::ast::LitKind::Number(0u64.into()), + }; + crate::mutation::mutant::MutationType::Assignment( + crate::mutation::visitor::AssignVarTypes::Literal( + lit_kind, + ), + ) + } + MutationDtoKindRead::AssignmentIdentifier { ident } => { + crate::mutation::mutant::MutationType::Assignment( + crate::mutation::visitor::AssignVarTypes::Identifier( + ident, + ), + ) + } + MutationDtoKindRead::BinaryOp { op } => { + let kind = match op.as_str() { + "+" => solar_parse::ast::BinOpKind::Add, + "-" => solar_parse::ast::BinOpKind::Sub, + "*" => solar_parse::ast::BinOpKind::Mul, + "/" => solar_parse::ast::BinOpKind::Div, + "&" => solar_parse::ast::BinOpKind::BitAnd, + "|" => solar_parse::ast::BinOpKind::BitOr, + "^" => solar_parse::ast::BinOpKind::BitXor, + "&&" => solar_parse::ast::BinOpKind::And, + "||" => solar_parse::ast::BinOpKind::Or, + "==" => solar_parse::ast::BinOpKind::Eq, + "!=" => solar_parse::ast::BinOpKind::Ne, + ">" => solar_parse::ast::BinOpKind::Gt, + ">=" => solar_parse::ast::BinOpKind::Ge, + "<" => solar_parse::ast::BinOpKind::Lt, + "<=" => solar_parse::ast::BinOpKind::Le, + other => panic!( + "Unknown binary operator token in cache: {}", + other + ), + }; + crate::mutation::mutant::MutationType::BinaryOp(kind) + } + MutationDtoKindRead::DeleteExpression => { + crate::mutation::mutant::MutationType::DeleteExpression + } + MutationDtoKindRead::ElimDelegate => { + crate::mutation::mutant::MutationType::ElimDelegate + } + MutationDtoKindRead::FunctionCall => { + crate::mutation::mutant::MutationType::FunctionCall + } + MutationDtoKindRead::Require => { + crate::mutation::mutant::MutationType::Require + } + MutationDtoKindRead::SwapArgumentsFunction => { + crate::mutation::mutant::MutationType::SwapArgumentsFunction + } + MutationDtoKindRead::SwapArgumentsOperator => { + crate::mutation::mutant::MutationType::SwapArgumentsOperator + } + MutationDtoKindRead::UnaryOperator { expr, op } => { + let resulting = match op.as_str() { + "PreInc" => solar_parse::ast::UnOpKind::PreInc, + "PostInc" => solar_parse::ast::UnOpKind::PostInc, + "PreDec" => solar_parse::ast::UnOpKind::PreDec, + "PostDec" => solar_parse::ast::UnOpKind::PostDec, + "Not" => solar_parse::ast::UnOpKind::Not, + "BitNot" => solar_parse::ast::UnOpKind::BitNot, + other => panic!( + "Unknown unary operator token in cache: {}", + other + ), + }; + crate::mutation::mutant::MutationType::UnaryOperator( + crate::mutation::mutant::UnaryOpMutated::new( + expr, resulting, + ), + ) + } + }; + out.push(Mutant { path: PathBuf::from(m.path), span, mutation }); + } + return Some(out); + } + } + } + // If the mutants file doesn't exist, treat as cache miss + return None; + } else { + // Stale entry: remove from mapping file + let mut updated = mapping.clone(); + updated.remove(&contract_path); + if let Ok(json) = serde_json::to_string_pretty(&updated) { + let _ = std::fs::create_dir_all(&mutation_cache_dir); + let _ = std::fs::write(&mapping_path, json); + } + return None; + } + } + + None + } + + /// Retrieves cached results for given build hash. + pub fn retrieve_cached_mutant_results( + &self, + hash: &str, + ) -> Option> { + let mutation_cache_dir = self.config.root.join(&self.config.mutation_dir); + let mapping_path = mutation_cache_dir.join("mapping.json"); + + let mapping: HashMap = match std::fs::read_to_string(&mapping_path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + { + Some(map) => map, + None => return None, + }; + + let contract_path = dunce::canonicalize(&self.contract_to_mutate) + .unwrap_or_else(|_| self.contract_to_mutate.clone()) + .to_string_lossy() + .into_owned(); + + if let Some(stored_hash) = mapping.get(&contract_path) { + if stored_hash == hash { + let contract_name = self + .contract_to_mutate + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("contract") + .to_string(); + let results_file = mutation_cache_dir.join(format!("{contract_name}.results")); + if results_file.exists() { + if let Ok(data) = std::fs::read_to_string(&results_file) { + #[derive(Deserialize)] + struct ResultDto { + path: String, + lo: u64, + hi: u64, + status: String, + } + + if let Ok(entries) = serde_json::from_str::>(&data) { + let mut out = Vec::with_capacity(entries.len()); + for e in entries { + let span = solar_parse::ast::Span::new( + BytePos(e.lo as u32), + BytePos(e.hi as u32), + ); + let status = match e.status.as_str() { + "dead" => crate::mutation::mutant::MutationResult::Dead, + "alive" => crate::mutation::mutant::MutationResult::Alive, + _ => crate::mutation::mutant::MutationResult::Invalid, + }; + // We need the full mutation to be able to reuse; find it via + // mutants cache if available + // Fallback: create a placeholder minimal Mutant with empty mutation + // (should not happen since we also cache full mutants) + // Here we try to match from cached mutants file + if let Some(mutants) = self.retrieve_cached_mutants(hash) { + if let Some(m) = mutants.into_iter().find(|m| { + m.path == PathBuf::from(&e.path) + && m.span.lo().0 as u64 == e.lo + && m.span.hi().0 as u64 == e.hi + }) { + out.push((m, status)); + continue; + } + } + out.push(( + Mutant { + path: PathBuf::from(e.path), + span, + mutation: + crate::mutation::mutant::MutationType::DeleteExpression, + }, + status, + )); + } + return Some(out); + } + } + } + } + } + None + } +} diff --git a/crates/forge/src/mutation/mutant.rs b/crates/forge/src/mutation/mutant.rs new file mode 100644 index 0000000000000..72b4001b64936 --- /dev/null +++ b/crates/forge/src/mutation/mutant.rs @@ -0,0 +1,203 @@ +// Generate mutants then run tests (reuse the whole unit test flow for now, including compilation to +// select mutants) Use Solar: +use super::visitor::AssignVarTypes; +use solar_interface::SourceMap; +use solar_parse::ast::{BinOpKind, LitKind, Span, UnOpKind}; +use std::{fmt::Display, path::PathBuf}; + +/// Wraps an unary operator mutated, to easily store pre/post-fix op swaps +#[derive(Debug, Clone)] +pub struct UnaryOpMutated { + /// String containing the whole new expression (operator and its target) + /// eg `a++` + new_expression: String, + + /// The underlying operator used by this mutant + pub resulting_op_kind: UnOpKind, +} + +impl UnaryOpMutated { + pub fn new(new_expression: String, resulting_op_kind: UnOpKind) -> Self { + Self { new_expression, resulting_op_kind } + } +} + +impl Display for UnaryOpMutated { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.new_expression) + } +} + +// @todo add a mutation from universalmutator: line swap (swap two lines of code, as it +// could theoretically uncover untested reentrancies +#[derive(Debug, Clone)] +pub enum MutationType { + // @todo Solar doesn't differentiate numeric type in LitKind (only on declaration?) -> for + // now, planket and let solc filter out the invalid mutants -> we might/should add a + // hashtable of the var to their underlying type (signed or not) so we avoid *a lot* of + // invalid mutants + /// For an initializer x, of type + /// bool: replace x with !x + /// uint: replace x with 0 + /// int: replace x with 0; replace x with -x (temp: this is mutated for uint as well) + /// + /// For a binary op y: apply BinaryOp(y) + Assignment(AssignVarTypes), + + /// For a binary op y in BinOpKind ("+", "-", ">=", etc) + /// replace y with each non-y in op + BinaryOp(BinOpKind), + + /// For a delete expr x `delete foo`, replace x with `assert(true)` + DeleteExpression, + + /// replace "delegatecall" with "call" + ElimDelegate, + + /// Gambit doesn't implement nor define it? + FunctionCall, + + // /// For a if(x) condition x: + // /// replace x with true; replace x with false + // This mutation is not used anymore, as we mutate the condition as an expression, + // which will creates true/false mutant as well as more complex conditions (eg if(foo++ > + // --bar) ) IfStatementMutation, + /// For a require(x) condition: + /// replace x with true; replace x with false + // Same as for IfStatementMutation, the expression inside the require is mutated as an + // expression to handle increment etc + Require, + + // @todo review if needed -> this might creates *a lot* of combinations for super-polyadic fn + // tho only swapping same type (to avoid obvious compilation failure), but should + // take into account implicit casting too... + /// For 2 args of the same type x,y in a function args: + /// swap(x, y) + SwapArgumentsFunction, + + // @todo same remark as above, might end up in a space too big to explore + filtering out + // based on type + /// For an expr taking 2 expression x, y (x+y, x-y, x = x + ...): + /// swap(x, y) + SwapArgumentsOperator, + + /// For an unary operator x in UnOpKind (eg "++", "--", "~", "!"): + /// replace x with all other operator in op + /// Pre or post- are different UnOp + UnaryOperator(UnaryOpMutated), +} + +impl MutationType { + fn get_name(&self) -> String { + match self { + Self::Assignment(var_type) => match var_type { + AssignVarTypes::Literal(kind) => { + format!("{}_{}", "Assignment", kind.description()) + } + AssignVarTypes::Identifier(ident) => { + format!("{}_{}", "Assignment", ident) + } + }, + Self::BinaryOp(kind) => { + format!("{}_{:?}", "BinaryOp", kind) + } + Self::DeleteExpression => "DeleteExpression".to_string(), + Self::ElimDelegate => "ElimDelegate".to_string(), + Self::FunctionCall => "FunctionCall".to_string(), + Self::Require => "Require".to_string(), + Self::SwapArgumentsFunction => "SwapArgumentsFunction".to_string(), + Self::SwapArgumentsOperator => "SwapArgumentsOperator".to_string(), + Self::UnaryOperator(mutated) => { + // avoid operator in tmp dir name + format!("{}_{:?}", "UnaryOperator", mutated.resulting_op_kind) + } + } + } +} + +impl Display for MutationType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Assignment(kind) => match kind { + AssignVarTypes::Literal(kind) => match kind { + LitKind::Number(val) => write!(f, "{val}"), + LitKind::Bool(val) => write!(f, "{val}"), + LitKind::Address(val) => write!(f, "{val}"), + // Reachable? + LitKind::Rational(val) => write!(f, "{val}"), + LitKind::Str(_, val, _) => write!(f, "{val:?}"), + LitKind::Err(val) => todo!(), + }, + AssignVarTypes::Identifier(ident) => write!(f, "{ident}"), + }, + Self::BinaryOp(kind) => write!(f, "{}", kind.to_str()), + Self::DeleteExpression => write!(f, "assert(true)"), + Self::ElimDelegate => write!(f, "call"), + Self::UnaryOperator(mutated) => write!(f, "{mutated}"), + + Self::FunctionCall + | Self::Require + | Self::SwapArgumentsFunction + | Self::SwapArgumentsOperator => write!(f, ""), + } + } +} + +#[derive(Debug)] +pub enum MutationResult { + Dead, + Alive, + Invalid, +} + +/// A given mutation +#[derive(Debug, Clone)] +pub struct Mutant { + /// The path to the project root where this mutant (tries to) live + pub path: PathBuf, + pub span: Span, + pub mutation: MutationType, +} + +impl Display for Mutant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}:{}-{}:{}", + self.path.display(), + self.span.lo().0, + self.span.hi().0, + self.mutation + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use solar_parse::ast::{BinOpKind, LitKind, Span, UnOpKind}; + + #[test] + fn test_mutation_type_get_name() { + assert_eq!(MutationType::DeleteExpression.get_name(), "DeleteExpression"); + assert_eq!(MutationType::ElimDelegate.get_name(), "ElimDelegate"); + assert_eq!(MutationType::FunctionCall.get_name(), "FunctionCall"); + assert_eq!(MutationType::Require.get_name(), "Require"); + assert_eq!(MutationType::SwapArgumentsFunction.get_name(), "SwapArgumentsFunction"); + assert_eq!(MutationType::SwapArgumentsOperator.get_name(), "SwapArgumentsOperator"); + + assert_eq!(MutationType::BinaryOp(BinOpKind::Add).get_name(), "BinaryOp_Add"); + + let lit_num = LitKind::Number(123.into()); + assert_eq!( + MutationType::Assignment(AssignVarTypes::Literal(lit_num)).get_name(), + "Assignment_number" + ); + + let ident = AssignVarTypes::Identifier("myVar".to_string()); + assert_eq!(MutationType::Assignment(ident).get_name(), "Assignment_myVar"); + + let unary_mutated = UnaryOpMutated::new("a--".to_string(), UnOpKind::PreInc); + assert_eq!(MutationType::UnaryOperator(unary_mutated).get_name(), "UnaryOperator_PreInc"); + } +} diff --git a/crates/forge/src/mutation/mutators/assignement_mutator.rs b/crates/forge/src/mutation/mutators/assignement_mutator.rs new file mode 100644 index 0000000000000..b6f963c150dc7 --- /dev/null +++ b/crates/forge/src/mutation/mutators/assignement_mutator.rs @@ -0,0 +1,113 @@ +use crate::mutation::{ + mutant::{Mutant, MutationType}, + mutators::{MutationContext, Mutator}, + visitor::AssignVarTypes, +}; + +use eyre::Result; +use solar_parse::ast::{Expr, ExprKind, LitKind, Span}; +use std::path::PathBuf; + +pub struct AssignmentMutator; + +impl Mutator for AssignmentMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let (assign_var_type, replacement_span) = match extract_rhs_info(context) { + Some(info) => info, + None => return Ok(vec![]), // is_applicable should filter this + }; + + match assign_var_type { + AssignVarTypes::Literal(lit) => match lit { + LitKind::Bool(val) => Ok(vec![Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal(LitKind::Bool( + !val, + ))), + path: context.path.clone(), + }]), + LitKind::Number(val) => Ok(vec![ + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + LitKind::Number(num_bigint::BigInt::ZERO), + )), + path: context.path.clone(), + }, + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + LitKind::Number(-val), + )), + path: context.path.clone(), + }, + ]), + _ => { + eyre::bail!("AssignmentMutator: unhandled literal kind on RHS: {:?}", lit) + } + }, + AssignVarTypes::Identifier(ident) => Ok(vec![ + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal(LitKind::Number( + num_bigint::BigInt::ZERO, + ))), + path: context.path.clone(), + }, + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Identifier(format!( + "-{ident}" + ))), + path: context.path.clone(), + }, + ]), + } + } + + /// Match is the expr is an assign with a var definiton having a literal or identifier as + /// initializer + fn is_applicable(&self, context: &MutationContext<'_>) -> bool { + if let Some(expr) = context.expr { + if let ExprKind::Assign(_lhs, _op_opt, rhs_actual_expr) = &expr.kind { + matches!((&**rhs_actual_expr).kind, ExprKind::Lit(..) | ExprKind::Ident(..)) + } else { + false // Not an assign + } + } else if let Some(var_definition) = context.var_definition { + if let Some(init) = &var_definition.initializer { + matches!(&init.kind, ExprKind::Lit(..) | ExprKind::Ident(..)) + } else { + false // No initializer + } + } else { + false // Not an expression or var_definition + } + } +} + +fn extract_rhs_info(context: &MutationContext<'_>) -> Option<(AssignVarTypes, Span)> { + let relevant_expr_for_rhs = if let Some(var_definition) = context.var_definition { + var_definition.initializer.as_ref()? + } else if let Some(expr) = context.expr { + match &expr.kind { + ExprKind::Assign(_lhs, _op_opt, rhs_actual_expr) => &**rhs_actual_expr, + // If the context.expr is already what we want to get the type from + // (e.g. a simple Lit or Ident being passed directly, though is_applicable filters this) + ExprKind::Lit(..) | ExprKind::Ident(..) => expr, + _ => return None, + } + } else { + return None; // No var_definition or expr in context (shouldn't happen?) + }; + + match &relevant_expr_for_rhs.kind { + ExprKind::Lit(kind, _) => { + Some((AssignVarTypes::Literal(kind.kind.clone()), relevant_expr_for_rhs.span)) + } + ExprKind::Ident(val) => { + Some((AssignVarTypes::Identifier(val.to_string()), relevant_expr_for_rhs.span)) + } + _ => None, + } +} diff --git a/crates/forge/src/mutation/mutators/binary_op_mutator.rs b/crates/forge/src/mutation/mutators/binary_op_mutator.rs new file mode 100644 index 0000000000000..4ceb2785c4be0 --- /dev/null +++ b/crates/forge/src/mutation/mutators/binary_op_mutator.rs @@ -0,0 +1,82 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; +use eyre::{OptionExt, Result}; +use solar_parse::ast::{BinOp, BinOpKind, ExprKind}; +use std::path::PathBuf; + +pub struct BinaryOpMutator; + +// @todo Add the other way to get there + +impl Mutator for BinaryOpMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let bin_op = get_bin_op(context)?; + let op = bin_op.kind; + + let operations_bools = vec![ + // Bool + BinOpKind::Lt, + BinOpKind::Le, + BinOpKind::Gt, + BinOpKind::Ge, + BinOpKind::Eq, + BinOpKind::Ne, + BinOpKind::Or, + BinOpKind::And, + ]; // this cover the "if" mutations, as every other mutant is tested, at least once + // @todo to optimize -> replace whole stmt (need new visitor override for visit_stmt tho) + // with true/false and skip operations_bools here (mayve some "level"/depth of + // mutation as param?) + + let operations_num_bitwise = vec![ + // Arithm + BinOpKind::Shr, + BinOpKind::Shl, + BinOpKind::Sar, + BinOpKind::BitAnd, + BinOpKind::BitOr, + BinOpKind::BitXor, + BinOpKind::Add, + BinOpKind::Sub, + BinOpKind::Pow, + BinOpKind::Mul, + BinOpKind::Div, + BinOpKind::Rem, + ]; + + let operations = + if operations_bools.contains(&op) { operations_bools } else { operations_num_bitwise }; + + Ok(operations + .into_iter() + .filter(|&kind| kind != op) + .map(|kind| Mutant { + span: context.span, + mutation: MutationType::BinaryOp(kind), + path: context.path.clone(), + }) + .collect()) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if ctxt.expr.is_none() { + return false; + } + + match ctxt.expr.unwrap().kind { + ExprKind::Binary(_, _, _) => true, + ExprKind::Assign(_, bin_op, _) => bin_op.is_some(), + _ => false, + } + } +} + +fn get_bin_op(ctxt: &MutationContext<'_>) -> Result { + let expr = ctxt.expr.ok_or_eyre("BinaryOpMutator: unexpected expression")?; + + match expr.kind { + ExprKind::Assign(_, Some(bin_op), _) => Ok(bin_op), + ExprKind::Binary(_, op, _) => Ok(op), + _ => eyre::bail!("BinaryOpMutator: unexpected expression kind"), + } +} diff --git a/crates/forge/src/mutation/mutators/delete_expression_mutator.rs b/crates/forge/src/mutation/mutators/delete_expression_mutator.rs new file mode 100644 index 0000000000000..8447bc69ed71a --- /dev/null +++ b/crates/forge/src/mutation/mutators/delete_expression_mutator.rs @@ -0,0 +1,22 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; +use solar_parse::ast::ExprKind; + +use eyre::Result; +use std::path::PathBuf; + +pub struct DeleteExpressionMutator; + +impl Mutator for DeleteExpressionMutator { + fn generate_mutants(&self, ctxt: &MutationContext<'_>) -> Result> { + Ok(vec![Mutant { + span: ctxt.span, + mutation: MutationType::DeleteExpression, + path: ctxt.path.clone(), + }]) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if let Some(expr) = ctxt.expr { matches!(expr.kind, ExprKind::Delete(_)) } else { false } + } +} diff --git a/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs b/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs new file mode 100644 index 0000000000000..741e26eb27729 --- /dev/null +++ b/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs @@ -0,0 +1,38 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; + +use eyre::Result; +use solar_parse::ast::ExprKind; +use std::{fmt::Display, path::PathBuf}; + +pub struct ElimDelegateMutator; + +impl Mutator for ElimDelegateMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + Ok(vec![Mutant { + span: context.span, + mutation: MutationType::ElimDelegate, + path: context.path.clone(), + }]) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + ctxt.expr + .as_ref() + .and_then(|expr| match &expr.kind { + ExprKind::Call(callee, _) => Some(callee), + _ => None, + }) + .and_then(|callee| match &callee.kind { + ExprKind::Member(_, ident) => Some(ident), + _ => None, + }) + .is_some_and(|ident| ident.to_string() == "delegatecall") + } +} + +impl Display for ElimDelegateMutator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "") + } +} diff --git a/crates/forge/src/mutation/mutators/mod.rs b/crates/forge/src/mutation/mutators/mod.rs new file mode 100644 index 0000000000000..b2a09e117d5e2 --- /dev/null +++ b/crates/forge/src/mutation/mutators/mod.rs @@ -0,0 +1,84 @@ +pub mod assignement_mutator; +pub mod binary_op_mutator; +pub mod delete_expression_mutator; +pub mod elim_delegate_mutator; +pub mod unary_op_mutator; + +pub mod mutator_registry; + +use eyre::Result; +use solar_parse::ast::{Expr, Span, VariableDefinition}; +use std::path::PathBuf; + +use crate::mutation::Mutant; + +pub trait Mutator { + /// Generate all mutant corresponding to a given context + fn generate_mutants(&self, ctxt: &MutationContext<'_>) -> Result>; + /// True if a mutator can be applied to an expression/node + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool; +} + +#[derive(Debug)] +pub struct MutationContext<'a> { + pub path: PathBuf, + pub span: Span, + /// The expression to mutate + pub expr: Option<&'a Expr<'a>>, + + pub var_definition: Option<&'a VariableDefinition<'a>>, +} + +impl<'a> MutationContext<'a> { + pub fn builder() -> MutationContextBuilder<'a> { + MutationContextBuilder::new() + } +} + +pub struct MutationContextBuilder<'a> { + path: Option, + span: Option, + expr: Option<&'a Expr<'a>>, + var_definition: Option<&'a VariableDefinition<'a>>, +} + +impl<'a> MutationContextBuilder<'a> { + // Create a new empty builder + pub fn new() -> Self { + MutationContextBuilder { path: None, span: None, expr: None, var_definition: None } + } + + // Required + pub fn with_path(mut self, path: PathBuf) -> Self { + self.path = Some(path); + self + } + + // Required + pub fn with_span(mut self, span: Span) -> Self { + self.span = Some(span); + self + } + + // Optional + pub fn with_expr(mut self, expr: &'a Expr<'a>) -> Self { + self.expr = Some(expr); + self + } + + // Optional + pub fn with_var_definition(mut self, var_definition: &'a VariableDefinition<'a>) -> Self { + self.var_definition = Some(var_definition); + self + } + + pub fn build(self) -> Result, &'static str> { + let span = self.span.ok_or("Span is required for MutationContext")?; + let path = self.path.ok_or("Path is required for MutationContext")?; + + Ok(MutationContext { path, span, expr: self.expr, var_definition: self.var_definition }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/forge/src/mutation/mutators/mutator_registry.rs b/crates/forge/src/mutation/mutators/mutator_registry.rs new file mode 100644 index 0000000000000..c24321458dcf6 --- /dev/null +++ b/crates/forge/src/mutation/mutators/mutator_registry.rs @@ -0,0 +1,40 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::Mutant; + +use super::{ + assignement_mutator, binary_op_mutator, delete_expression_mutator, elim_delegate_mutator, + unary_op_mutator, +}; + +/// Registry of all available mutators (ie implementing the Mutator trait) +pub struct MutatorRegistry { + mutators: Vec>, +} + +impl MutatorRegistry { + pub fn default() -> Self { + let mut registry = Self { mutators: Vec::new() }; + + registry.mutators.push(Box::new(assignement_mutator::AssignmentMutator)); + registry.mutators.push(Box::new(binary_op_mutator::BinaryOpMutator)); + registry.mutators.push(Box::new(delete_expression_mutator::DeleteExpressionMutator)); + registry.mutators.push(Box::new(elim_delegate_mutator::ElimDelegateMutator)); + registry.mutators.push(Box::new(unary_op_mutator::UnaryOperatorMutator)); + + registry + } + + pub fn new_with_mutators(mutators: Vec>) -> Self { + Self { mutators } + } + + /// Find all applicable mutators for a given context and return the corresponding mutations + pub fn generate_mutations(&self, context: &MutationContext<'_>) -> Vec { + self.mutators + .iter() + .filter(|mutator| mutator.is_applicable(context)) + .filter_map(|mutator| mutator.generate_mutants(context).ok()) + .flatten() + .collect() + } +} diff --git a/crates/forge/src/mutation/mutators/tests/assignement_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/assignement_mutator_test.rs new file mode 100644 index 0000000000000..ff11e843d2009 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/assignement_mutator_test.rs @@ -0,0 +1,24 @@ +use crate::mutation::mutators::{ + assignement_mutator::AssignmentMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for AssignmentMutator {} + +#[rstest] +#[case::assign_lit("x = y", Some(vec!["x = 0", "x = -y"]))] +#[case::assign_number("x = 123", Some(vec!["x = 0", "x = -123"]))] +#[case::assign_bool("x = true", Some(vec!["x = false"]))] +#[case::assign_bool("x = false", Some(vec!["x = true"]))] +#[case::assign_declare("uint256 x = 123", Some(vec!["uint256 x = 0", "uint256 x = -123"]))] +#[case::non_assign("a = b + c", None)] +fn test_mutator_assignment( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: AssignmentMutator = AssignmentMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + AssignmentMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs new file mode 100644 index 0000000000000..84a874d017043 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs @@ -0,0 +1,31 @@ +use crate::mutation::mutators::{ + binary_op_mutator::BinaryOpMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for BinaryOpMutator {} + +#[rstest] +#[case::add("x + y", Some(vec!["x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::sub("x - y", Some(vec!["x + y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::mul("x * y", Some(vec!["x + y", "x - y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::div("x / y", Some(vec!["x + y", "x - y", "x * y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::modulus("x % y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::pow("x ** y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_shift_left("x << y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_shift_right("x >> y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_shift_right_unsigned("x >>> y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_and("x & y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x | y", "x ^ y"]))] +#[case::bit_or("x | y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x ^ y"]))] +#[case::bit_xor("x ^ y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y"]))] +#[case::non_binary("a = true", None)] +fn test_mutator_bitwise( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: BinaryOpMutator = BinaryOpMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + BinaryOpMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs new file mode 100644 index 0000000000000..ba94e5295837d --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs @@ -0,0 +1,20 @@ +use crate::mutation::mutators::{ + delete_expression_mutator::DeleteExpressionMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for DeleteExpressionMutator {} + +#[rstest] +#[case::delete_expr("delete x", Some(vec!["x"]))] +#[case::non_delete("a = b + c", None)] +fn test_mutator_delete_expr( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: DeleteExpressionMutator = DeleteExpressionMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + DeleteExpressionMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs new file mode 100644 index 0000000000000..05828beb24822 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs @@ -0,0 +1,20 @@ +use crate::mutation::mutators::{ + elim_delegate_mutator::ElimDelegateMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for ElimDelegateMutator {} + +#[rstest] +#[case::delegate_expr("address(this).delegatecall{value: 1 ether}(0)", Some(vec!["address(this).call{value: 1 ether}(0)"]))] +#[case::non_delegate("address(this).call{value: 1 ether}(0)", None)] +fn test_mutator_delegate_expr( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: ElimDelegateMutator = ElimDelegateMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + ElimDelegateMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/helper.rs b/crates/forge/src/mutation/mutators/tests/helper.rs new file mode 100644 index 0000000000000..7a9a4e177e489 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/helper.rs @@ -0,0 +1,55 @@ +use crate::mutation::{mutators::Mutator, visitor::MutantVisitor, Session}; +use solar_parse::{ + ast::{interface::source_map::FileName, visit::Visit, Arena}, + Parser, +}; + +use std::path::PathBuf; +pub struct MutatorTestCase<'a> { + /// @dev needs to be in a function, to avoid parsing error from solar + /// eg `let input = "function f() { x = 1; }"` to test x = 1 + pub input: &'a str, + /// All the mutations expected for this input, using this mutator + pub expected_mutations: Option>, +} + +pub trait MutatorTester { + fn test_mutator(mutator: M, test_case: MutatorTestCase<'_>) { + let arena = Arena::new(); + let sess = Session::builder().with_silent_emitter(None).build(); + + // let mut mutations: Vec = Vec::new(); + let mut mutant_visitor = MutantVisitor::new_with_mutators(vec![Box::new(mutator)]); + + let _ = sess.enter(|| -> solar_parse::interface::Result<()> { + let mut parser = Parser::from_lazy_source_code( + &sess, + &arena, + FileName::Real(PathBuf::from(test_case.input)), + || Ok(test_case.input.to_string()), + )?; + + let ast = parser.parse_file().map_err(|e| e.emit())?; + + mutant_visitor.visit_source_unit(&ast); + + let mutations = mutant_visitor.mutation_to_conduct; + + // @todo test mutants content... + if let Some(expected) = test_case.expected_mutations { + assert_eq!(mutations.len(), expected.len()); + + for mutation in mutations { + assert!(expected.contains(&mutation.mutation.to_string().as_str())); + } + } else { + assert_eq!(mutations.len(), 0); + } + + Ok(()) + }); + } +} + +// Implement for unit test module +impl MutatorTester for () {} diff --git a/crates/forge/src/mutation/mutators/tests/mod.rs b/crates/forge/src/mutation/mutators/tests/mod.rs new file mode 100644 index 0000000000000..403ee2be95bd9 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/mod.rs @@ -0,0 +1,10 @@ +mod assignement_mutator_test; +mod binary_op_mutator_test; + +mod delete_expression_mutator_test; + +mod elim_delegate_mutator_test; + +mod helper; + +mod unary_op_mutator_test; diff --git a/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs new file mode 100644 index 0000000000000..4c7ece82de2da --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs @@ -0,0 +1,26 @@ +use crate::mutation::mutators::{ + tests::helper::{MutatorTestCase, MutatorTester}, + unary_op_mutator::UnaryOperatorMutator, +}; + +use rstest::*; + +impl MutatorTester for UnaryOperatorMutator {} + +#[rstest] +#[case::pre_inc("++x", Some(vec!["--x", "~x", "-x", "x++", "x--"]))] +#[case::pre_dec("--x", Some(vec!["++x", "~x", "-x", "x++", "x--"]))] +#[case::neg("-x", Some(vec!["++x", "--x", "~x", "x++", "x--"]))] +#[case::bit_not("~x", Some(vec!["++x", "--x", "-x", "x++", "x--"]))] +#[case::post_inc("x++",Some(vec!["++x", "--x", "~x", "-x", "x--"]))] +#[case::post_dec("x--",Some(vec!["++x", "--x", "~x", "-x", "x++"]))] +#[case::bool("!x", Some(vec!["x"]))] +#[case::non_unary("a = b + c", None)] +fn test_unary_op_mutator_arithmetic( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: UnaryOperatorMutator = UnaryOperatorMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + UnaryOperatorMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/unary_op_mutator.rs b/crates/forge/src/mutation/mutators/unary_op_mutator.rs new file mode 100644 index 0000000000000..b8e4315f357f2 --- /dev/null +++ b/crates/forge/src/mutation/mutators/unary_op_mutator.rs @@ -0,0 +1,107 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType, UnaryOpMutated}; +use eyre::Result; +use solar_parse::ast::{ExprKind, LitKind, UnOpKind}; +use std::path::PathBuf; + +pub struct UnaryOperatorMutator; + +impl Mutator for UnaryOperatorMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let operations = vec![ + UnOpKind::PreInc, // number + UnOpKind::PreDec, // n + UnOpKind::Neg, // n @todo filter this one only for int + UnOpKind::BitNot, // n + ]; + + let post_fixed_operations = vec![UnOpKind::PostInc, UnOpKind::PostDec]; + + let expr = context.expr.unwrap(); + + let target_kind; + let op; + + match &expr.kind { + ExprKind::Unary(un_op, target) => { + target_kind = &target.kind; + op = un_op.kind; + } + _ => unreachable!(), + }; + + let target_content = match target_kind { + ExprKind::Lit(lit, _) => match &lit.kind { + LitKind::Bool(val) => val.to_string(), + LitKind::Number(val) => val.to_string(), + _ => String::new(), + }, + ExprKind::Ident(inner) => inner.to_string(), + ExprKind::Member(expr, ident) => { + match expr.kind { + ExprKind::Ident(inner) => { + format!("{}{}", ident.as_str(), inner.to_string()) + } // @todo not supporting something like a.b[0]++ + _ => String::new(), + } + } + _ => String::new(), + }; + + // Bool has only the Not operator as possible target -> we try removing it + if op == UnOpKind::Not { + return Ok(vec![Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(UnaryOpMutated::new( + target_content.to_string(), + UnOpKind::Not, + )), + path: context.path.clone(), + }]); + } + + let mut mutations: Vec; + + mutations = operations + .into_iter() + .filter(|&kind| kind != op) + .map(|kind| { + let new_expression = format!("{}{}", kind.to_str(), target_content); + + let mutated = UnaryOpMutated::new(new_expression, kind); + + Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(mutated), + path: context.path.clone(), + } + }) + .collect(); + + mutations.extend(post_fixed_operations.into_iter().filter(|&kind| kind != op).map( + |kind| { + let new_expression = format!("{}{}", target_content, kind.to_str()); + + let mutated = UnaryOpMutated::new(new_expression, kind); + + Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(mutated), + path: context.path.clone(), + } + }, + )); + + Ok(mutations) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if let Some(expr) = ctxt.expr { + if let ExprKind::Unary(_, _) = &expr.kind { + return true; + } + } + + false + } +} diff --git a/crates/forge/src/mutation/reporter.rs b/crates/forge/src/mutation/reporter.rs new file mode 100644 index 0000000000000..b6c3a78e383c1 --- /dev/null +++ b/crates/forge/src/mutation/reporter.rs @@ -0,0 +1,55 @@ +use crate::mutation::MutationsSummary; +use comfy_table::{Attribute, Cell, Color, Row, Table, modifiers::UTF8_ROUND_CORNERS}; +pub struct MutationReporter { + table: Table, +} + +impl MutationReporter { + pub fn new() -> Self { + let mut table = Table::new(); + + table.apply_modifier(UTF8_ROUND_CORNERS); + + table.set_header(vec![ + Cell::new("Status"), + Cell::new("# Mutants"), + Cell::new("% of Total"), + ]); + + Self { table } + } + + pub fn report(&mut self, summary: &MutationsSummary) { + let mut row = Row::new(); + row.add_cell(Cell::new("Survived").fg(Color::Red)) + .add_cell(Cell::new(summary.total_survived().to_string())) + .add_cell(Cell::new(format!( + "{:.2}%", + summary.total_survived() as f64 / summary.total_mutants() as f64 * 100. + ))); + self.table.add_row(row); + + row = Row::new(); + row.add_cell(Cell::new("Dead").fg(Color::Green)) + .add_cell(Cell::new(summary.total_dead().to_string())) + .add_cell(Cell::new(format!( + "{:.2}%", + summary.total_dead() as f64 / summary.total_mutants() as f64 * 100. + ))); + self.table.add_row(row); + + row = Row::new(); + row.add_cell(Cell::new("Invalid").fg(Color::Green)) + .add_cell(Cell::new(summary.total_invalid().to_string())) + .add_cell(Cell::new(format!( + "{:.2}%", + summary.total_invalid() as f64 / summary.total_mutants() as f64 * 100. + ))); + self.table.add_row(row); + + sh_println!("Total number of mutants generated: {}", summary.total_mutants()); + sh_println!("\n{}\n", self.table); + sh_println!("Dead mutants: {}\n", summary.dead()); + sh_println!("Survived mutants: {}\n", summary.survived()); + } +} diff --git a/crates/forge/src/mutation/visitor.rs b/crates/forge/src/mutation/visitor.rs new file mode 100644 index 0000000000000..c353720fc6334 --- /dev/null +++ b/crates/forge/src/mutation/visitor.rs @@ -0,0 +1,69 @@ +use crate::mutation::mutators::Mutator; +use solar_parse::ast::{Expr, LitKind, SourceUnit, VariableDefinition, visit::Visit}; +use std::{ops::ControlFlow, path::PathBuf}; + +use crate::mutation::{ + mutant::Mutant, + mutators::{MutationContext, mutator_registry::MutatorRegistry}, +}; + +#[derive(Debug, Clone)] +pub enum AssignVarTypes { + Literal(LitKind), + Identifier(String), /* not using Ident as the symbol is slow to convert as to_str() <-- + * maybe will have to switch back if validating more aggressively */ +} + +/// A visitor which collect all expression to mutate as well as the mutation types +pub struct MutantVisitor { + pub mutation_to_conduct: Vec, + pub mutator_registry: MutatorRegistry, + pub path: PathBuf, +} + +impl MutantVisitor { + /// Use all mutator from registry::default + pub fn default(path: PathBuf) -> Self { + Self { mutation_to_conduct: Vec::new(), mutator_registry: MutatorRegistry::default(), path } + } + + /// Use only a set of mutators + pub fn new_with_mutators(path: PathBuf, mutators: Vec>) -> Self { + Self { + mutation_to_conduct: Vec::new(), + mutator_registry: MutatorRegistry::new_with_mutators(mutators), + path, + } + } +} + +impl<'ast> Visit<'ast> for MutantVisitor { + type BreakValue = (); + + fn visit_variable_definition( + &mut self, + var: &'ast VariableDefinition<'ast>, + ) -> ControlFlow { + let context = MutationContext::builder() + .with_path(self.path.clone()) + .with_span(var.span) + .with_var_definition(var) + .build() + .unwrap(); + + self.mutation_to_conduct.extend(self.mutator_registry.generate_mutations(&context)); + self.walk_variable_definition(var) + } + + fn visit_expr(&mut self, expr: &'ast Expr<'ast>) -> ControlFlow { + let context = MutationContext::builder() + .with_path(self.path.clone()) + .with_span(expr.span) + .with_expr(expr) + .build() + .unwrap(); + + self.mutation_to_conduct.extend(self.mutator_registry.generate_mutations(&context)); + self.walk_expr(expr) + } +} diff --git a/crates/forge/tests/cli/config.rs b/crates/forge/tests/cli/config.rs index aa27ba775495c..f9a448382c9cc 100644 --- a/crates/forge/tests/cli/config.rs +++ b/crates/forge/tests/cli/config.rs @@ -77,6 +77,7 @@ forgetest!(can_extract_config_values, |prj, cmd| { path_pattern_inverse: None, coverage_pattern_inverse: None, test_failures_file: "test-cache/test-failures".into(), + mutation_dir: "test-cache/mutation".into(), threads: None, show_progress: false, fuzz: FuzzConfig { diff --git a/crates/forge/tests/it/main.rs b/crates/forge/tests/it/main.rs index c8890af3d4614..45e30e91583bb 100644 --- a/crates/forge/tests/it/main.rs +++ b/crates/forge/tests/it/main.rs @@ -8,6 +8,7 @@ mod fs; mod fuzz; mod inline; mod invariant; +mod mutation; mod repros; mod spec; mod table; diff --git a/crates/forge/tests/it/mutation.rs b/crates/forge/tests/it/mutation.rs new file mode 100644 index 0000000000000..fda00f2eb6181 --- /dev/null +++ b/crates/forge/tests/it/mutation.rs @@ -0,0 +1,118 @@ +use forge::mutation::MutationHandler; +use forge_script::ScriptArgs; +use foundry_common::shell::{ColorChoice, OutputFormat, OutputMode, Shell}; +use std::sync::Arc; + +#[tokio::test(flavor = "multi_thread")] +async fn test_mutation_test_lifecycle() { + let contract = r#" + // SPDX-License-Identifier: UNLICENSED + pragma solidity ^0.8.13; + + contract Counter { + uint256 public number; + + function increment() public { + number++; + // This should result in 5 mutants: ++number, --number, -number, ~number, number-- + // -number should be invalid + // ++number should be alive + // the rest should be dead + } + }"#; + + let test = r#" + // SPDX-License-Identifier: UNLICENSED + pragma solidity ^0.8.13; + + // Avoid having to manage a libs folder + import {Counter} from "../src/Counter.sol"; + + contract CounterTest { + Counter public counter; + + function setUp() public { + counter = new Counter(); + } + + function test_Increment() public { + uint256 _countBefore = counter.number(); + + counter.increment(); + + assert(counter.number() == _countBefore + 1); + } + }"#; + + let temp_dir = tempfile::tempdir().unwrap(); + + let src_dir = temp_dir.path().join("src"); + std::fs::create_dir_all(&src_dir).expect("Failed to create src directory"); + + let test_dir = temp_dir.path().join("test"); + std::fs::create_dir_all(&test_dir).expect("Failed to create test directory"); + + let cache_dir = temp_dir.path().join("cache"); + std::fs::create_dir_all(&cache_dir).expect("Failed to create test directory"); + + let out_dir = temp_dir.path().join("out"); + std::fs::create_dir_all(&out_dir).expect("Failed to create test directory"); + + std::fs::write(&src_dir.join("Counter.sol"), contract) + .unwrap_or_else(|_| panic!("Failed to write to target file {:?}", &src_dir)); + + std::fs::write(&test_dir.join("CounterTest.t.sol"), test) + .unwrap_or_else(|_| panic!("Failed to write to target file {:?}", &src_dir)); + + let mut config = foundry_config::Config::default(); + config.cache_path = cache_dir; + config.out = out_dir; + config.src = src_dir.clone(); + config.test = test_dir.clone(); + + let mut mutation_handler = MutationHandler::new(src_dir.join("Counter.sol"), Arc::new(config)); + + mutation_handler.read_source_contract(); + mutation_handler.generate_ast().await; + mutation_handler.create_mutation_folders(); + let mutants = mutation_handler.generate_and_compile().await; + + // Test if we compile and collect the valid/invalid mutants + assert_eq!(mutants.iter().filter(|(_, output)| output.is_none()).count(), 1); + assert_eq!(mutants.iter().filter(|(_, output)| output.is_some()).count(), 4); + + // @todo run the tests + let mut invalids = 0; + let mut alive = 0; + let mut dead = 0; + + // Create a new shell to suppress any script output + let shell = Shell::new_with(OutputFormat::Json, OutputMode::Quiet, ColorChoice::Never, 0); + shell.set(); + + // Run the tests as scripts, for convenience + for mutant in mutants { + if mutant.1.is_some() { + let result = ScriptArgs { + path: mutant.0.path.join("test/CounterTest.t.sol").to_string_lossy().to_string(), + sig: "test_Increment".to_string(), + args: vec![], + ..Default::default() + } + .run_script() + .await; + + if result.is_err() { + dead += 1; + } else { + alive += 1; + } + } else { + invalids += 1; + } + } + + assert_eq!(invalids, 1); + assert_eq!(alive, 1); + assert_eq!(dead, 3); +}