diff --git a/Cargo.lock b/Cargo.lock index c7d4bd20f4f..1766a2fe069 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3414,6 +3414,7 @@ dependencies = [ "noirc_abi", "noirc_evaluator", "noirc_frontend", + "proptest", ] [[package]] diff --git a/tooling/ast_fuzzer/fuzz/Cargo.toml b/tooling/ast_fuzzer/fuzz/Cargo.toml index 01065a81a4b..98ba5d58347 100644 --- a/tooling/ast_fuzzer/fuzz/Cargo.toml +++ b/tooling/ast_fuzzer/fuzz/Cargo.toml @@ -26,6 +26,7 @@ noir_ast_fuzzer = { path = ".." } [dev-dependencies] arbtest.workspace = true env_logger.workspace = true +proptest.workspace = true [[bin]] diff --git a/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs b/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs index c441ed1662f..a2677486e13 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs @@ -52,6 +52,7 @@ mod tests { /// ``` #[test] fn fuzz_with_arbtest() { - crate::targets::tests::fuzz_with_arbtest(super::fuzz); + // TODO: Allow more tests when the 1510th case is fixed. + crate::targets::tests::fuzz_with_arbtest(super::fuzz, 1509); } } diff --git a/tooling/ast_fuzzer/fuzz/src/targets/comptime_vs_brillig.rs b/tooling/ast_fuzzer/fuzz/src/targets/comptime_vs_brillig.rs index 121dae55c24..e069ec08494 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/comptime_vs_brillig.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/comptime_vs_brillig.rs @@ -56,6 +56,7 @@ mod tests { /// ``` #[test] fn fuzz_with_arbtest() { - crate::targets::tests::fuzz_with_arbtest(super::fuzz); + // TODO(#8870): Allow more tests when the bug is fixed (fails in the 128th case). + crate::targets::tests::fuzz_with_arbtest(super::fuzz, 127); } } diff --git a/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs b/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs index 56ce7e8e716..a55f7ef93a8 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs @@ -56,8 +56,6 @@ pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> { #[cfg(test)] mod tests { - use crate::targets::tests::is_running_in_ci; - /// ```ignore /// NOIR_ARBTEST_SEED=0x6819c61400001000 \ /// NOIR_AST_FUZZER_SHOW_AST=1 \ @@ -65,10 +63,6 @@ mod tests { /// ``` #[test] fn fuzz_with_arbtest() { - if is_running_in_ci() { - // TODO: Investigate second program constraint failures. - return; - } - crate::targets::tests::fuzz_with_arbtest(super::fuzz); + crate::targets::tests::fuzz_with_arbtest(super::fuzz, 2000); } } diff --git a/tooling/ast_fuzzer/fuzz/src/targets/mod.rs b/tooling/ast_fuzzer/fuzz/src/targets/mod.rs index 20c3b1791d2..7d811c78852 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/mod.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/mod.rs @@ -6,10 +6,16 @@ pub mod pass_vs_prev; #[cfg(test)] mod tests { + + const TIMEOUT: Duration = Duration::from_secs(20); + const MIN_SIZE: u32 = 1 << 12; + const MAX_SIZE: u32 = 1 << 20; + use std::time::Duration; use arbitrary::Unstructured; use color_eyre::eyre; + use proptest::prelude::*; pub fn seed_from_env() -> Option { let Ok(seed) = std::env::var("NOIR_ARBTEST_SEED") else { return None }; @@ -18,14 +24,7 @@ mod tests { Some(seed) } - /// We can use this to disable the proptests on CI until we fix known bugs. - /// - /// The tests should always be enabled locally. They can be run with: - /// - /// ```ignore - /// cargo test -p noir_ast_fuzzer_fuzz - /// ``` - #[allow(unused)] + /// Check if we are running on CI. pub fn is_running_in_ci() -> bool { std::env::var("CI").is_ok() } @@ -39,21 +38,72 @@ mod tests { /// NOIR_AST_FUZZER_SHOW_AST=1 \ /// cargo test -p noir_ast_fuzzer_fuzz acir_vs_brillig /// ``` - pub fn fuzz_with_arbtest(f: impl Fn(&mut Unstructured) -> eyre::Result<()>) { + /// + /// The `cases` determine how many tests to run on CI. + /// Tune this so that we can expect CI to be able to get through all cases in reasonable time. + pub fn fuzz_with_arbtest(f: impl Fn(&mut Unstructured) -> eyre::Result<()>, cases: u32) { let _ = env_logger::try_init(); - let mut prop = arbtest::arbtest(|u| { + if let Some(seed) = seed_from_env() { + run_reproduce(f, seed); + } else if is_running_in_ci() { + run_deterministic(f, cases); + } else { + run_nondeterministic(f); + } + } + + /// Reproduce the result of a single seed. + fn run_reproduce(f: impl Fn(&mut Unstructured) -> eyre::Result<()>, seed: u64) { + arbtest::arbtest(|u| { f(u).unwrap(); Ok(()) }) - .budget(Duration::from_secs(20)) - .size_min(1 << 12) - .size_max(1 << 20); + .seed(seed) + .run(); + } - if let Some(seed) = seed_from_env() { - prop = prop.seed(seed); - } + /// Run the tests non-deterministically until the timeout. + /// + /// This is the local behavior. + fn run_nondeterministic(f: impl Fn(&mut Unstructured) -> eyre::Result<()>) { + arbtest::arbtest(|u| { + f(u).unwrap(); + Ok(()) + }) + .size_min(MIN_SIZE) + .size_max(MAX_SIZE) + .budget(TIMEOUT) + .run(); + } + + /// Run multiple tests with a deterministic RNG. + /// + /// This is the behavior on CI. + fn run_deterministic(f: impl Fn(&mut Unstructured) -> eyre::Result<()>, cases: u32) { + let config = proptest::test_runner::Config { + cases, + failure_persistence: None, + max_shrink_iters: 0, + ..Default::default() + }; + let rng = proptest::test_runner::TestRng::deterministic_rng(config.rng_algorithm); + let mut runner = proptest::test_runner::TestRunner::new_with_rng(config, rng); + + runner + .run(&seed_strategy(), |seed| { + run_reproduce(&f, seed); + Ok(()) + }) + .unwrap(); + } - prop.run(); + /// Generate seeds for `arbtest` where the top 32 bits are random and the lower 32 bits represent the input size. + fn seed_strategy() -> proptest::strategy::BoxedStrategy { + (MIN_SIZE..MAX_SIZE) + .prop_flat_map(move |size| { + any::().prop_map(move |raw| (size as u64) | (raw << u32::BITS)) + }) + .boxed() } } diff --git a/tooling/ast_fuzzer/fuzz/src/targets/orig_vs_morph.rs b/tooling/ast_fuzzer/fuzz/src/targets/orig_vs_morph.rs index c7078d4a2f1..dbcdefce2f4 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/orig_vs_morph.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/orig_vs_morph.rs @@ -363,6 +363,6 @@ mod tests { /// ``` #[test] fn fuzz_with_arbtest() { - crate::targets::tests::fuzz_with_arbtest(super::fuzz); + crate::targets::tests::fuzz_with_arbtest(super::fuzz, 2000); } } diff --git a/tooling/ast_fuzzer/fuzz/src/targets/pass_vs_prev.rs b/tooling/ast_fuzzer/fuzz/src/targets/pass_vs_prev.rs index d6ca6ca6c72..936f40e9b76 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/pass_vs_prev.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/pass_vs_prev.rs @@ -71,6 +71,7 @@ mod tests { /// ``` #[test] fn fuzz_with_arbtest() { - crate::targets::tests::fuzz_with_arbtest(super::fuzz); + // TODO: Allow more cases when the 3180th case is fixed. + crate::targets::tests::fuzz_with_arbtest(super::fuzz, 3179); } }