diff --git a/.github/workflows/nightly-fuzz-test.yml b/.github/workflows/nightly-fuzz-test.yml new file mode 100644 index 00000000000..2c05f711289 --- /dev/null +++ b/.github/workflows/nightly-fuzz-test.yml @@ -0,0 +1,37 @@ +name: Non-deterministic fuzz tests +on: + workflow_dispatch: + schedule: + # Run nightly at 0300 + - cron: "0 3 * * *" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # On regular PRs we run deterministic fuzzing to avoid flaky tests on CI. + # In the nightly tests we want to explore uncharted territory. + NOIR_AST_FUZZER_FORCE_NON_DETERMINISTIC: 1 + # Tell arbtest how long it can run the tests for. + NOIR_AST_FUZZER_FORCE_BUDGET_SECS: 300 + +jobs: + ast-fuzz: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup toolchain + uses: dtolnay/rust-toolchain@1.85.0 + with: + targets: x86_64-unknown-linux-gnu + + - name: Run tests + run: cargo test -p noir_ast_fuzzer_fuzz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/tooling/ast_fuzzer/fuzz/src/lib.rs b/tooling/ast_fuzzer/fuzz/src/lib.rs index 2ee78b57421..4fd77063757 100644 --- a/tooling/ast_fuzzer/fuzz/src/lib.rs +++ b/tooling/ast_fuzzer/fuzz/src/lib.rs @@ -16,13 +16,17 @@ use noirc_frontend::monomorphization::ast::Program; pub mod targets; +fn bool_from_env(key: &str) -> bool { + std::env::var(key).map(|s| s == "1" || s == "true").unwrap_or_default() +} + // TODO(#7876): Allow specifying options on the command line. fn show_ast() -> bool { - std::env::var("NOIR_AST_FUZZER_SHOW_AST").map(|s| s == "1" || s == "true").unwrap_or_default() + bool_from_env("NOIR_AST_FUZZER_SHOW_AST") } fn show_ssa() -> bool { - std::env::var("NOIR_AST_FUZZER_SHOW_SSA").map(|s| s == "1" || s == "true").unwrap_or_default() + bool_from_env("NOIR_AST_FUZZER_SHOW_SSA") } pub fn default_ssa_options() -> SsaEvaluatorOptions { diff --git a/tooling/ast_fuzzer/fuzz/src/targets/mod.rs b/tooling/ast_fuzzer/fuzz/src/targets/mod.rs index 7d811c78852..0f57694748e 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/mod.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/mod.rs @@ -7,7 +7,7 @@ pub mod pass_vs_prev; #[cfg(test)] mod tests { - const TIMEOUT: Duration = Duration::from_secs(20); + const BUDGET: Duration = Duration::from_secs(20); const MIN_SIZE: u32 = 1 << 12; const MAX_SIZE: u32 = 1 << 20; @@ -17,7 +17,9 @@ mod tests { use color_eyre::eyre; use proptest::prelude::*; - pub fn seed_from_env() -> Option { + use crate::bool_from_env; + + fn seed_from_env() -> Option { let Ok(seed) = std::env::var("NOIR_ARBTEST_SEED") else { return None }; let seed = u64::from_str_radix(seed.trim_start_matches("0x"), 16) .unwrap_or_else(|e| panic!("failed to parse seed '{seed}': {e}")); @@ -25,10 +27,23 @@ mod tests { } /// Check if we are running on CI. - pub fn is_running_in_ci() -> bool { + fn is_running_in_ci() -> bool { std::env::var("CI").is_ok() } + /// Check if we explicitly want non-deterministic behavior, even on CI. + fn force_non_deterministic() -> bool { + bool_from_env("NOIR_AST_FUZZER_FORCE_NON_DETERMINISTIC") + } + + /// How long to let non-deterministic tests run for. + fn budget() -> Duration { + std::env::var("NOIR_AST_FUZZER_BUDGET_SECS").ok().map_or(BUDGET, |b| { + let secs = b.parse().unwrap_or_else(|e| panic!("failed to parse budget; got {b}: {e}")); + Duration::from_secs(secs) + }) + } + /// `cargo fuzz` takes a long time to ramp up the complexity. /// This test catches crash bugs much faster. /// @@ -46,7 +61,7 @@ mod tests { if let Some(seed) = seed_from_env() { run_reproduce(f, seed); - } else if is_running_in_ci() { + } else if is_running_in_ci() && !force_non_deterministic() { run_deterministic(f, cases); } else { run_nondeterministic(f); @@ -73,7 +88,7 @@ mod tests { }) .size_min(MIN_SIZE) .size_max(MAX_SIZE) - .budget(TIMEOUT) + .budget(budget()) .run(); }