diff --git a/.github/workflows/nightly-fuzz-test.yml b/.github/workflows/nightly-fuzz-test.yml index 2c05f711289..a1560d52d14 100644 --- a/.github/workflows/nightly-fuzz-test.yml +++ b/.github/workflows/nightly-fuzz-test.yml @@ -16,8 +16,8 @@ env: # On regular PRs we run deterministic fuzzing to avoid flaky tests on CI. # In the nightly tests we want to explore uncharted territory. NOIR_AST_FUZZER_FORCE_NON_DETERMINISTIC: 1 - # Tell arbtest how long it can run the tests for. - NOIR_AST_FUZZER_FORCE_BUDGET_SECS: 300 + # How long should we run the fuzzing for, in seconds. + NOIR_AST_FUZZER_BUDGET_SECS: 300 jobs: ast-fuzz: diff --git a/TESTING.md b/TESTING.md index a758ea610d8..e7ded32a0c3 100644 --- a/TESTING.md +++ b/TESTING.md @@ -104,7 +104,7 @@ cargo test -p noir_ast_fuzzer_fuzz arbtest Should it find any problems, the test would print a _seed_ which we can use to replicate the test, for example: ```shell -NOIR_ARBTEST_SEED=0x6819c61400001000 cargo test -p noir_ast_fuzzer_fuzz comptime_vs_brillig +NOIR_AST_FUZZER_SEED=0x6819c61400001000 cargo test -p noir_ast_fuzzer_fuzz comptime_vs_brillig ``` -When the `NOIR_ARBTEST_SEED` is present, or when there is a non-panicky error, the tests will print out the Noir AST and the `Prover.toml` file which can be used to reproduce the problem as an integration test. +When the `NOIR_AST_FUZZER_SEED` is present, or when there is a non-panicky error, the tests will print out the Noir AST and the `Prover.toml` file which can be used to reproduce the problem as an integration test. diff --git a/tooling/ast_fuzzer/README.md b/tooling/ast_fuzzer/README.md index 4c4f481e3c0..26ef2669f7f 100644 --- a/tooling/ast_fuzzer/README.md +++ b/tooling/ast_fuzzer/README.md @@ -48,7 +48,7 @@ To get quick feedback about whether there are any easy-to-discover bugs, we can cargo test -p noir_ast_fuzzer_fuzz arbtest ``` -Unlike `cargo fuzz`, these don't "ramp up" the complexity of the code, but go full tilt from the beginning, and only run for a limited amount of time (e.g. 10 seconds). Upon failure they print a hexadecimal `seed`, which can be used with the `NOIR_ARBTEST_SEED` env var to replicate the error. +Unlike `cargo fuzz`, these don't "ramp up" the complexity of the code, but go full tilt from the beginning, and only run for a limited amount of time (e.g. 10 seconds). Upon failure they print a hexadecimal `seed`, which can be used with the `NOIR_AST_FUZZER_SEED` env var to replicate the error. ## Minimizing Noir diff --git a/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs b/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs index fb9684e9835..aabfa46445a 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs @@ -46,7 +46,7 @@ pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> { mod tests { /// ```ignore - /// NOIR_ARBTEST_SEED=0x6819c61400001000 \ + /// NOIR_AST_FUZZER_SEED=0x6819c61400001000 \ /// NOIR_AST_FUZZER_SHOW_AST=1 \ /// cargo test -p noir_ast_fuzzer_fuzz acir_vs_brillig /// ``` diff --git a/tooling/ast_fuzzer/fuzz/src/targets/comptime_vs_brillig.rs b/tooling/ast_fuzzer/fuzz/src/targets/comptime_vs_brillig.rs index b79a10759a5..849ac73bd90 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/comptime_vs_brillig.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/comptime_vs_brillig.rs @@ -50,7 +50,7 @@ pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> { mod tests { /// ```ignore - /// NOIR_ARBTEST_SEED=0x6819c61400001000 \ + /// NOIR_AST_FUZZER_SEED=0x6819c61400001000 \ /// NOIR_AST_FUZZER_SHOW_AST=1 \ /// cargo test -p noir_ast_fuzzer_fuzz comptime_vs_brillig /// ``` diff --git a/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs b/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs index a55f7ef93a8..364e8a83450 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs @@ -57,7 +57,7 @@ pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> { #[cfg(test)] mod tests { /// ```ignore - /// NOIR_ARBTEST_SEED=0x6819c61400001000 \ + /// NOIR_AST_FUZZER_SEED=0x6819c61400001000 \ /// NOIR_AST_FUZZER_SHOW_AST=1 \ /// cargo test -p noir_ast_fuzzer_fuzz min_vs_full /// ``` diff --git a/tooling/ast_fuzzer/fuzz/src/targets/mod.rs b/tooling/ast_fuzzer/fuzz/src/targets/mod.rs index 0f57694748e..bb06ff6e980 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/mod.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/mod.rs @@ -20,12 +20,21 @@ mod tests { use crate::bool_from_env; fn seed_from_env() -> Option { - let Ok(seed) = std::env::var("NOIR_ARBTEST_SEED") else { return None }; + let Ok(seed) = std::env::var("NOIR_AST_FUZZER_SEED") else { return None }; let seed = u64::from_str_radix(seed.trim_start_matches("0x"), 16) .unwrap_or_else(|e| panic!("failed to parse seed '{seed}': {e}")); Some(seed) } + /// How long to let non-deterministic tests run for. + fn budget() -> Duration { + std::env::var("NOIR_AST_FUZZER_BUDGET_SECS").ok().map_or(BUDGET, |b| { + Duration::from_secs( + b.parse().unwrap_or_else(|e| panic!("failed to parse budget; got {b}: {e}")), + ) + }) + } + /// Check if we are running on CI. fn is_running_in_ci() -> bool { std::env::var("CI").is_ok() @@ -36,20 +45,12 @@ mod tests { bool_from_env("NOIR_AST_FUZZER_FORCE_NON_DETERMINISTIC") } - /// How long to let non-deterministic tests run for. - fn budget() -> Duration { - std::env::var("NOIR_AST_FUZZER_BUDGET_SECS").ok().map_or(BUDGET, |b| { - let secs = b.parse().unwrap_or_else(|e| panic!("failed to parse budget; got {b}: {e}")); - Duration::from_secs(secs) - }) - } - /// `cargo fuzz` takes a long time to ramp up the complexity. /// This test catches crash bugs much faster. /// /// Run it with for example: /// ```ignore - /// NOIR_ARBTEST_SEED=0x6819c61400001000 \ + /// NOIR_AST_FUZZER_SEED=0x6819c61400001000 \ /// NOIR_AST_FUZZER_SHOW_AST=1 \ /// cargo test -p noir_ast_fuzzer_fuzz acir_vs_brillig /// ``` diff --git a/tooling/ast_fuzzer/fuzz/src/targets/orig_vs_morph.rs b/tooling/ast_fuzzer/fuzz/src/targets/orig_vs_morph.rs index 48664365af6..66f2cc23fb4 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/orig_vs_morph.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/orig_vs_morph.rs @@ -806,7 +806,7 @@ mod helpers { #[cfg(test)] mod tests { /// ```ignore - /// NOIR_ARBTEST_SEED=0xb2fb5f0b00100000 \ + /// NOIR_AST_FUZZER_SEED=0xb2fb5f0b00100000 \ /// NOIR_AST_FUZZER_SHOW_AST=1 \ /// cargo test -p noir_ast_fuzzer_fuzz orig_vs_morph /// ``` diff --git a/tooling/ast_fuzzer/fuzz/src/targets/pass_vs_prev.rs b/tooling/ast_fuzzer/fuzz/src/targets/pass_vs_prev.rs index 3f7b2f25022..4e73fe50821 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/pass_vs_prev.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/pass_vs_prev.rs @@ -65,7 +65,7 @@ fn clone_ssa(ssa: &Ssa) -> Ssa { #[cfg(test)] mod tests { /// ```ignore - /// NOIR_ARBTEST_SEED=0x6819c61400001000 \ + /// NOIR_AST_FUZZER_SEED=0x6819c61400001000 \ /// NOIR_AST_FUZZER_SHOW_AST=1 \ /// cargo test -p noir_ast_fuzzer_fuzz pass_vs_prev /// ``` diff --git a/tooling/ast_fuzzer/src/compare/compiled.rs b/tooling/ast_fuzzer/src/compare/compiled.rs index 79a99e68720..c5043fbf354 100644 --- a/tooling/ast_fuzzer/src/compare/compiled.rs +++ b/tooling/ast_fuzzer/src/compare/compiled.rs @@ -184,6 +184,13 @@ impl

CompareCompiled

{ let blackbox_solver = Bn254BlackBoxSolver(false); let initial_witness = self.abi.encode(&self.input_map, None).wrap_err("abi::encode")?; + log::debug!( + "ABI input:\n{}", + noirc_abi::input_parser::Format::Toml + .serialize(&self.input_map, &self.abi) + .unwrap_or_else(|e| format!("failed to serialize inputs: {e}")) + ); + let do_exec = |program| { let mut print = Vec::new(); diff --git a/tooling/ast_fuzzer/tests/parser.rs b/tooling/ast_fuzzer/tests/parser.rs index 76b4d15891d..7be3cb9b2f7 100644 --- a/tooling/ast_fuzzer/tests/parser.rs +++ b/tooling/ast_fuzzer/tests/parser.rs @@ -19,7 +19,7 @@ use noirc_evaluator::{ }; fn seed_from_env() -> Option { - let Ok(seed) = std::env::var("NOIR_ARBTEST_SEED") else { return None }; + let Ok(seed) = std::env::var("NOIR_AST_FUZZER_SEED") else { return None }; let seed = u64::from_str_radix(seed.trim_start_matches("0x"), 16) .unwrap_or_else(|e| panic!("failed to parse seed '{seed}': {e}")); Some(seed) diff --git a/tooling/ast_fuzzer/tests/smoke.rs b/tooling/ast_fuzzer/tests/smoke.rs index 9ff846c5d54..48917c93679 100644 --- a/tooling/ast_fuzzer/tests/smoke.rs +++ b/tooling/ast_fuzzer/tests/smoke.rs @@ -17,7 +17,7 @@ use noirc_abi::input_parser::Format; use noirc_evaluator::{brillig::BrilligOptions, ssa}; fn seed_from_env() -> Option { - let Ok(seed) = std::env::var("NOIR_ARBTEST_SEED") else { return None }; + let Ok(seed) = std::env::var("NOIR_AST_FUZZER_SEED") else { return None }; let seed = u64::from_str_radix(seed.trim_start_matches("0x"), 16) .unwrap_or_else(|e| panic!("failed to parse seed '{seed}': {e}")); Some(seed)