diff --git a/tooling/ast_fuzzer/fuzz/src/lib.rs b/tooling/ast_fuzzer/fuzz/src/lib.rs index fdcb48fd186..99f7132273c 100644 --- a/tooling/ast_fuzzer/fuzz/src/lib.rs +++ b/tooling/ast_fuzzer/fuzz/src/lib.rs @@ -1,7 +1,7 @@ use acir::circuit::ExpressionWidth; use color_eyre::eyre; use noir_ast_fuzzer::DisplayAstAsNoir; -use noir_ast_fuzzer::compare::{CompareComptime, CompareResult, CompareSsa, HasPrograms}; +use noir_ast_fuzzer::compare::{CompareCompiled, CompareComptime, CompareResult, HasPrograms}; use noirc_abi::input_parser::Format; use noirc_evaluator::brillig::Brillig; use noirc_evaluator::ssa::{SsaPass, primary_passes, secondary_passes}; @@ -79,9 +79,9 @@ where } /// Compare the execution result and print the inputs if the result is a failure. -pub fn compare_results

(inputs: &CompareSsa

, result: &CompareResult) -> eyre::Result<()> +pub fn compare_results

(inputs: &CompareCompiled

, result: &CompareResult) -> eyre::Result<()> where - CompareSsa

: HasPrograms, + CompareCompiled

: HasPrograms, { let res = result.return_value_or_err(); diff --git a/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs b/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs index e65bad960da..8334c6065d8 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs @@ -5,7 +5,7 @@ use arbitrary::Arbitrary; use arbitrary::Unstructured; use color_eyre::eyre; use noir_ast_fuzzer::Config; -use noir_ast_fuzzer::compare::{CompareOptions, ComparePasses}; +use noir_ast_fuzzer::compare::{CompareOptions, ComparePipelines}; use noir_ast_fuzzer::rewrite::change_all_functions_into_unconstrained; pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> { @@ -19,7 +19,7 @@ pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> { ..Default::default() }; - let inputs = ComparePasses::arb( + let inputs = ComparePipelines::arb( u, config, |u, program| { diff --git a/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs b/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs index 8f1ae7cc3ca..1cea5996521 100644 --- a/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs +++ b/tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs @@ -6,7 +6,7 @@ use crate::{ }; use arbitrary::{Arbitrary, Unstructured}; use color_eyre::eyre; -use noir_ast_fuzzer::compare::{CompareOptions, ComparePasses}; +use noir_ast_fuzzer::compare::{CompareOptions, ComparePipelines}; use noir_ast_fuzzer::{ Config, compare::CompareResult, rewrite::change_all_functions_into_unconstrained, }; @@ -23,7 +23,7 @@ pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> { ..Default::default() }; - let inputs = ComparePasses::arb( + let inputs = ComparePipelines::arb( u, config, |_u, program| { diff --git a/tooling/ast_fuzzer/src/compare.rs b/tooling/ast_fuzzer/src/compare.rs deleted file mode 100644 index 81891b73eed..00000000000 --- a/tooling/ast_fuzzer/src/compare.rs +++ /dev/null @@ -1,401 +0,0 @@ -use std::collections::BTreeMap; -use std::path::Path; - -use acir::{FieldElement, native_types::WitnessStack}; -use acvm::pwg::{OpcodeResolutionError, ResolvedAssertionPayload}; -use arbitrary::{Arbitrary, Unstructured}; -use bn254_blackbox_solver::Bn254BlackBoxSolver; -use color_eyre::eyre::{self, WrapErr, bail}; -use nargo::{ - NargoError, errors::ExecutionError, foreign_calls::DefaultForeignCallBuilder, parse_all, -}; -use noirc_abi::{Abi, InputMap, input_parser::InputValue}; -use noirc_driver::{ - CompilationResult, CompileOptions, CompiledProgram, CrateId, compile_main, - file_manager_with_stdlib, prepare_crate, -}; -use noirc_evaluator::ssa::{SsaEvaluatorOptions, SsaProgramArtifact}; -use noirc_frontend::{hir::Context, monomorphization::ast::Program}; - -use crate::{ - Config, DisplayAstAsNoirComptime, arb_inputs, arb_program, arb_program_comptime, program_abi, -}; - -#[derive(Clone, Debug, PartialEq)] -pub struct ExecOutput { - pub return_value: Option, - pub print_output: String, -} - -type ExecResult = (Result, NargoError>, String); - -/// Prepare a code snippet. -/// (copied from nargo_cli/tests/common.rs) -fn prepare_snippet(source: String) -> (Context<'static, 'static>, CrateId) { - let root = Path::new(""); - let file_name = Path::new("main.nr"); - let mut file_manager = file_manager_with_stdlib(root); - file_manager.add_file_with_source(file_name, source).expect( - "Adding source buffer to file manager should never fail when file manager is empty", - ); - let parsed_files = parse_all(&file_manager); - - let mut context = Context::new(file_manager, parsed_files); - let root_crate_id = prepare_crate(&mut context, file_name); - - (context, root_crate_id) -} - -/// Compile the main function in a code snippet. -/// -/// Use `force_brillig` to test it as an unconstrained function without having to change the code. -/// This is useful for methods that use the `runtime::is_unconstrained()` method to change their behavior. -/// (copied from nargo_cli/tests/common.rs) -pub fn prepare_and_compile_snippet( - source: String, - force_brillig: bool, -) -> CompilationResult { - let (mut context, root_crate_id) = prepare_snippet(source); - let options = CompileOptions { - force_brillig, - silence_warnings: true, - skip_underconstrained_check: true, - skip_brillig_constraints_check: true, - ..Default::default() - }; - compile_main(&mut context, root_crate_id, &options, None) -} - -/// Subset of [SsaEvaluatorOptions] that we want to vary. -/// -/// It exists to reduce noise in the printed results, compared to showing the full `SsaEvaluatorOptions`. -#[derive(Debug, Clone, Default)] -pub struct CompareOptions { - pub inliner_aggressiveness: i64, -} - -impl Arbitrary<'_> for CompareOptions { - fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { - Ok(Self { inliner_aggressiveness: *u.choose(&[i64::MIN, 0, i64::MAX])? }) - } -} - -impl CompareOptions { - /// Copy fields into an [SsaEvaluatorOptions] instance. - pub fn onto(&self, mut options: SsaEvaluatorOptions) -> SsaEvaluatorOptions { - options.inliner_aggressiveness = self.inliner_aggressiveness; - options - } -} - -/// Possible outcomes of the differential execution of two equivalent programs. -/// -/// Use [CompareResult::return_value_or_err] to do the final comparison between -/// the execution result. -pub enum CompareResult { - BothFailed(NargoError, NargoError), - LeftFailed(NargoError, ExecOutput), - RightFailed(ExecOutput, NargoError), - BothPassed(ExecOutput, ExecOutput), -} - -impl CompareResult { - fn new( - abi: &Abi, - (res1, print1): ExecResult, - (res2, print2): ExecResult, - ) -> eyre::Result { - let decode = |ws: WitnessStack| -> eyre::Result> { - let wm = &ws.peek().expect("there should be a main witness").witness; - let (_, r) = abi.decode(wm).wrap_err("abi::decode")?; - Ok(r) - }; - - match (res1, res2) { - (Err(e1), Err(e2)) => Ok(CompareResult::BothFailed(e1, e2)), - (Err(e1), Ok(ws2)) => Ok(CompareResult::LeftFailed( - e1, - ExecOutput { return_value: decode(ws2)?, print_output: print2 }, - )), - (Ok(ws1), Err(e2)) => Ok(CompareResult::RightFailed( - ExecOutput { return_value: decode(ws1)?, print_output: print1 }, - e2, - )), - (Ok(ws1), Ok(ws2)) => { - let o1 = ExecOutput { return_value: decode(ws1)?, print_output: print1 }; - let o2 = ExecOutput { return_value: decode(ws2)?, print_output: print2 }; - Ok(CompareResult::BothPassed(o1, o2)) - } - } - } - - /// Check that the programs agree on a return value. - /// - /// Returns an error if anything is different. - pub fn return_value_or_err(&self) -> eyre::Result> { - match self { - CompareResult::BothFailed(e1, e2) => { - if Self::errors_match(e1, e2) { - // Both programs failed the same way. - Ok(None) - } else { - bail!("both programs failed: {e1} vs {e2}\n{e1:?}\n{e2:?}") - } - } - CompareResult::LeftFailed(e, _) => { - bail!("first program failed: {e}\n{e:?}") - } - CompareResult::RightFailed(_, e) => { - bail!("second program failed: {e}\n{e:?}") - } - CompareResult::BothPassed(o1, o2) => { - if o1.return_value != o2.return_value { - bail!( - "programs disagree on return value:\n{:?}\n!=\n{:?}", - o1.return_value, - o2.return_value - ) - } else if o1.print_output != o2.print_output { - bail!( - "programs disagree on printed output:\n---\n{}\n\n---\n{}\n", - o1.print_output, - o2.print_output - ) - } else { - Ok(o1.return_value.as_ref()) - } - } - } - } - - /// Check whether two errors can be considered equivalent. - fn errors_match(e1: &NargoError, e2: &NargoError) -> bool { - use ExecutionError::*; - - // For now consider non-execution errors as failures we need to investigate. - let NargoError::ExecutionError(ee1) = e1 else { - return false; - }; - let NargoError::ExecutionError(ee2) = e2 else { - return false; - }; - - match (ee1, ee2) { - (AssertionFailed(p1, _, _), AssertionFailed(p2, _, _)) => p1 == p2, - (SolvingError(s1, _), SolvingError(s2, _)) => format!("{s1}") == format!("{s2}"), - (SolvingError(s, _), AssertionFailed(p, _, _)) - | (AssertionFailed(p, _, _), SolvingError(s, _)) => match (s, p) { - ( - OpcodeResolutionError::UnsatisfiedConstrain { .. }, - ResolvedAssertionPayload::String(s), - ) => s == "Attempted to divide by zero", - _ => false, - }, - } - } -} - -pub struct CompareArtifact { - pub options: CompareOptions, - pub artifact: SsaProgramArtifact, -} - -impl CompareArtifact { - fn new(artifact: SsaProgramArtifact, options: CompareOptions) -> Self { - Self { artifact, options } - } -} - -impl From<(SsaProgramArtifact, CompareOptions)> for CompareArtifact { - fn from((artifact, options): (SsaProgramArtifact, CompareOptions)) -> Self { - Self::new(artifact, options) - } -} - -/// Compare the execution of a Noir program in pure comptime (via interpreter) -/// vs normal SSA execution. -pub struct CompareComptime { - pub program: Program, - pub abi: Abi, - pub source: String, - pub ssa: CompareArtifact, - pub force_brillig: bool, -} - -impl CompareComptime { - /// Execute the Noir code and the SSA, then compare the results. - pub fn exec(&self) -> eyre::Result { - println!("{}", self.source); - let program1 = match prepare_and_compile_snippet(self.source.clone(), self.force_brillig) { - Ok((program, _)) => program, - Err(e) => panic!("failed to compile program:\n{}\n{e:?}", self.source), - }; - - let blackbox_solver = Bn254BlackBoxSolver(false); - let initial_witness = self.abi.encode(&BTreeMap::new(), None).wrap_err("abi::encode")?; - - let do_exec = |program| { - let mut print = Vec::new(); - - let mut foreign_call_executor = DefaultForeignCallBuilder::default() - .with_mocks(false) - .with_output(&mut print) - .build(); - - let res = nargo::ops::execute_program( - program, - initial_witness.clone(), - &blackbox_solver, - &mut foreign_call_executor, - ); - - let print = String::from_utf8(print).expect("should be valid utf8 string"); - (res, print) - }; - - let (res1, print1) = do_exec(&program1.program); - let (res2, print2) = do_exec(&self.ssa.artifact.program); - - CompareResult::new(&self.abi, (res1, print1), (res2, print2)) - } - - /// Generate a random comptime-viable AST, reverse it into - /// Noir code and also compile it into SSA. - pub fn arb( - u: &mut Unstructured, - c: Config, - f: impl FnOnce( - &mut Unstructured, - Program, - ) -> arbitrary::Result<(SsaProgramArtifact, CompareOptions)>, - ) -> arbitrary::Result { - let force_brillig = c.force_brillig; - let program = arb_program_comptime(u, c)?; - let abi = program_abi(&program); - - let ssa = CompareArtifact::from(f(u, program.clone())?); - - let source = format!("{}", DisplayAstAsNoirComptime(&program)); - - Ok(Self { program, abi, source, ssa, force_brillig }) - } -} - -/// Compare the execution of different SSA representations of equivalent program(s). -pub struct CompareSsa

{ - pub program: P, - pub abi: Abi, - pub input_map: InputMap, - pub ssa1: CompareArtifact, - pub ssa2: CompareArtifact, -} - -impl

CompareSsa

{ - /// Execute the two SSAs and compare the results. - pub fn exec(&self) -> eyre::Result { - let blackbox_solver = Bn254BlackBoxSolver(false); - let initial_witness = self.abi.encode(&self.input_map, None).wrap_err("abi::encode")?; - - let do_exec = |program| { - let mut print = Vec::new(); - - let mut foreign_call_executor = DefaultForeignCallBuilder::default() - .with_mocks(false) - .with_output(&mut print) - .build(); - - let res = nargo::ops::execute_program( - program, - initial_witness.clone(), - &blackbox_solver, - &mut foreign_call_executor, - ); - - let print = String::from_utf8(print).expect("should be valid utf8 string"); - (res, print) - }; - - let (res1, print1) = do_exec(&self.ssa1.artifact.program); - let (res2, print2) = do_exec(&self.ssa2.artifact.program); - - CompareResult::new(&self.abi, (res1, print1), (res2, print2)) - } -} - -/// Compare the execution the same program compiled in two different ways. -pub type ComparePasses = CompareSsa; - -impl CompareSsa { - /// Generate a random AST and compile it into SSA in two different ways. - pub fn arb( - u: &mut Unstructured, - c: Config, - f: impl FnOnce( - &mut Unstructured, - Program, - ) -> arbitrary::Result<(SsaProgramArtifact, CompareOptions)>, - g: impl FnOnce( - &mut Unstructured, - Program, - ) -> arbitrary::Result<(SsaProgramArtifact, CompareOptions)>, - ) -> arbitrary::Result { - let program = arb_program(u, c)?; - let abi = program_abi(&program); - - let ssa1 = CompareArtifact::from(f(u, program.clone())?); - let ssa2 = CompareArtifact::from(g(u, program.clone())?); - - let input_program = &ssa1.artifact.program; - let input_map = arb_inputs(u, input_program, &abi)?; - - Ok(Self { program, abi, input_map, ssa1, ssa2 }) - } -} - -/// Compare two equivalent variants of the same program, compiled the same way. -pub type CompareMorph = CompareSsa<(Program, Program)>; - -impl CompareMorph { - /// Generate a random AST, a random metamorph of it, then compile both into SSA with the same options. - pub fn arb( - u: &mut Unstructured, - c: Config, - f: impl Fn(&mut Unstructured, Program) -> arbitrary::Result<(Program, CompareOptions)>, - g: impl Fn(Program, &CompareOptions) -> SsaProgramArtifact, - ) -> arbitrary::Result { - let program1 = arb_program(u, c)?; - let (program2, options) = f(u, program1.clone())?; - let abi = program_abi(&program1); - - let ssa1 = g(program1.clone(), &options); - let ssa2 = g(program2.clone(), &options); - - let input_program = &ssa1.program; - let input_map = arb_inputs(u, input_program, &abi)?; - - Ok(Self { - program: (program1, program2), - abi, - input_map, - ssa1: CompareArtifact::new(ssa1, options.clone()), - ssa2: CompareArtifact::new(ssa2, options), - }) - } -} - -/// Help iterate over the program(s) in the comparable artifact. -pub trait HasPrograms { - fn programs(&self) -> Vec<&Program>; -} - -impl HasPrograms for ComparePasses { - fn programs(&self) -> Vec<&Program> { - vec![&self.program] - } -} - -impl HasPrograms for CompareMorph { - fn programs(&self) -> Vec<&Program> { - vec![&self.program.0, &self.program.1] - } -} diff --git a/tooling/ast_fuzzer/src/compare/compiled.rs b/tooling/ast_fuzzer/src/compare/compiled.rs new file mode 100644 index 00000000000..6a373f18fa9 --- /dev/null +++ b/tooling/ast_fuzzer/src/compare/compiled.rs @@ -0,0 +1,142 @@ +use arbitrary::Unstructured; +use bn254_blackbox_solver::Bn254BlackBoxSolver; +use color_eyre::eyre::{self, WrapErr}; +use nargo::foreign_calls::DefaultForeignCallBuilder; +use noirc_abi::{Abi, InputMap}; +use noirc_evaluator::ssa::SsaProgramArtifact; +use noirc_frontend::monomorphization::ast::Program; + +use crate::{Config, arb_inputs, arb_program, program_abi}; + +use super::{CompareOptions, CompareResult, HasPrograms}; + +pub struct CompareArtifact { + pub options: CompareOptions, + pub artifact: SsaProgramArtifact, +} + +impl CompareArtifact { + fn new(artifact: SsaProgramArtifact, options: CompareOptions) -> Self { + Self { artifact, options } + } +} + +impl From<(SsaProgramArtifact, CompareOptions)> for CompareArtifact { + fn from((artifact, options): (SsaProgramArtifact, CompareOptions)) -> Self { + Self::new(artifact, options) + } +} + +/// Compare the execution of equivalent programs, compiled in different ways. +pub struct CompareCompiled

{ + pub program: P, + pub abi: Abi, + pub input_map: InputMap, + pub ssa1: CompareArtifact, + pub ssa2: CompareArtifact, +} + +impl

CompareCompiled

{ + /// Execute the two SSAs and compare the results. + pub fn exec(&self) -> eyre::Result { + let blackbox_solver = Bn254BlackBoxSolver(false); + let initial_witness = self.abi.encode(&self.input_map, None).wrap_err("abi::encode")?; + + let do_exec = |program| { + let mut print = Vec::new(); + + let mut foreign_call_executor = DefaultForeignCallBuilder::default() + .with_mocks(false) + .with_output(&mut print) + .build(); + + let res = nargo::ops::execute_program( + program, + initial_witness.clone(), + &blackbox_solver, + &mut foreign_call_executor, + ); + + let print = String::from_utf8(print).expect("should be valid utf8 string"); + (res, print) + }; + + let (res1, print1) = do_exec(&self.ssa1.artifact.program); + let (res2, print2) = do_exec(&self.ssa2.artifact.program); + + CompareResult::new(&self.abi, (res1, print1), (res2, print2)) + } +} + +/// Compare the execution the same program compiled in two different ways. +pub type ComparePipelines = CompareCompiled; + +impl CompareCompiled { + /// Generate a random AST and compile it into SSA in two different ways. + pub fn arb( + u: &mut Unstructured, + c: Config, + f: impl FnOnce( + &mut Unstructured, + Program, + ) -> arbitrary::Result<(SsaProgramArtifact, CompareOptions)>, + g: impl FnOnce( + &mut Unstructured, + Program, + ) -> arbitrary::Result<(SsaProgramArtifact, CompareOptions)>, + ) -> arbitrary::Result { + let program = arb_program(u, c)?; + let abi = program_abi(&program); + + let ssa1 = CompareArtifact::from(f(u, program.clone())?); + let ssa2 = CompareArtifact::from(g(u, program.clone())?); + + let input_program = &ssa1.artifact.program; + let input_map = arb_inputs(u, input_program, &abi)?; + + Ok(Self { program, abi, input_map, ssa1, ssa2 }) + } +} + +impl HasPrograms for ComparePipelines { + fn programs(&self) -> Vec<&Program> { + vec![&self.program] + } +} + +/// Compare two equivalent variants of the same program, compiled the same way. +pub type CompareMorph = CompareCompiled<(Program, Program)>; + +impl CompareMorph { + /// Generate a random AST, a random metamorph of it, then compile both into SSA with the same options. + pub fn arb( + u: &mut Unstructured, + c: Config, + f: impl Fn(&mut Unstructured, Program) -> arbitrary::Result<(Program, CompareOptions)>, + g: impl Fn(Program, &CompareOptions) -> SsaProgramArtifact, + ) -> arbitrary::Result { + let program1 = arb_program(u, c)?; + let (program2, options) = f(u, program1.clone())?; + let abi = program_abi(&program1); + + let ssa1 = g(program1.clone(), &options); + let ssa2 = g(program2.clone(), &options); + + let input_program = &ssa1.program; + let input_map = arb_inputs(u, input_program, &abi)?; + + Ok(Self { + program: (program1, program2), + abi, + input_map, + ssa1: CompareArtifact::new(ssa1, options.clone()), + ssa2: CompareArtifact::new(ssa2, options), + }) + } +} + +impl HasPrograms for CompareMorph { + fn programs(&self) -> Vec<&Program> { + vec![&self.program.0, &self.program.1] + } +} diff --git a/tooling/ast_fuzzer/src/compare/comptime.rs b/tooling/ast_fuzzer/src/compare/comptime.rs new file mode 100644 index 00000000000..9e437726063 --- /dev/null +++ b/tooling/ast_fuzzer/src/compare/comptime.rs @@ -0,0 +1,131 @@ +use std::collections::BTreeMap; +use std::path::Path; + +use arbitrary::Unstructured; +use bn254_blackbox_solver::Bn254BlackBoxSolver; +use color_eyre::eyre::{self, WrapErr}; +use nargo::{foreign_calls::DefaultForeignCallBuilder, parse_all}; +use noirc_abi::Abi; +use noirc_driver::{ + CompilationResult, CompileOptions, CompiledProgram, CrateId, compile_main, + file_manager_with_stdlib, prepare_crate, +}; +use noirc_evaluator::ssa::SsaProgramArtifact; +use noirc_frontend::{hir::Context, monomorphization::ast::Program}; + +use crate::{ + Config, DisplayAstAsNoirComptime, arb_program_comptime, compare::CompareResult, program_abi, +}; + +use super::{CompareArtifact, CompareOptions, HasPrograms}; + +/// Prepare a code snippet. +/// (copied from nargo_cli/tests/common.rs) +fn prepare_snippet(source: String) -> (Context<'static, 'static>, CrateId) { + let root = Path::new(""); + let file_name = Path::new("main.nr"); + let mut file_manager = file_manager_with_stdlib(root); + file_manager.add_file_with_source(file_name, source).expect( + "Adding source buffer to file manager should never fail when file manager is empty", + ); + let parsed_files = parse_all(&file_manager); + + let mut context = Context::new(file_manager, parsed_files); + let root_crate_id = prepare_crate(&mut context, file_name); + + (context, root_crate_id) +} + +/// Compile the main function in a code snippet. +/// +/// Use `force_brillig` to test it as an unconstrained function without having to change the code. +/// This is useful for methods that use the `runtime::is_unconstrained()` method to change their behavior. +/// (copied from nargo_cli/tests/common.rs) +fn prepare_and_compile_snippet( + source: String, + force_brillig: bool, +) -> CompilationResult { + let (mut context, root_crate_id) = prepare_snippet(source); + let options = CompileOptions { + force_brillig, + silence_warnings: true, + skip_underconstrained_check: true, + skip_brillig_constraints_check: true, + ..Default::default() + }; + compile_main(&mut context, root_crate_id, &options, None) +} + +/// Compare the execution of a Noir program in pure comptime (via interpreter) +/// vs normal SSA execution. +pub struct CompareComptime { + pub program: Program, + pub abi: Abi, + pub source: String, + pub ssa: CompareArtifact, + pub force_brillig: bool, +} + +impl CompareComptime { + /// Execute the Noir code and the SSA, then compare the results. + pub fn exec(&self) -> eyre::Result { + let program1 = match prepare_and_compile_snippet(self.source.clone(), self.force_brillig) { + Ok((program, _)) => program, + Err(e) => panic!("failed to compile program:\n{}\n{e:?}", self.source), + }; + + let blackbox_solver = Bn254BlackBoxSolver(false); + let initial_witness = self.abi.encode(&BTreeMap::new(), None).wrap_err("abi::encode")?; + + let do_exec = |program| { + let mut print = Vec::new(); + + let mut foreign_call_executor = DefaultForeignCallBuilder::default() + .with_mocks(false) + .with_output(&mut print) + .build(); + + let res = nargo::ops::execute_program( + program, + initial_witness.clone(), + &blackbox_solver, + &mut foreign_call_executor, + ); + + let print = String::from_utf8(print).expect("should be valid utf8 string"); + (res, print) + }; + + let (res1, print1) = do_exec(&program1.program); + let (res2, print2) = do_exec(&self.ssa.artifact.program); + + CompareResult::new(&self.abi, (res1, print1), (res2, print2)) + } + + /// Generate a random comptime-viable AST, reverse it into + /// Noir code and also compile it into SSA. + pub fn arb( + u: &mut Unstructured, + c: Config, + f: impl FnOnce( + &mut Unstructured, + Program, + ) -> arbitrary::Result<(SsaProgramArtifact, CompareOptions)>, + ) -> arbitrary::Result { + let force_brillig = c.force_brillig; + let program = arb_program_comptime(u, c)?; + let abi = program_abi(&program); + + let ssa = CompareArtifact::from(f(u, program.clone())?); + + let source = format!("{}", DisplayAstAsNoirComptime(&program)); + + Ok(Self { program, abi, source, ssa, force_brillig }) + } +} + +impl HasPrograms for CompareComptime { + fn programs(&self) -> Vec<&Program> { + vec![&self.program] + } +} diff --git a/tooling/ast_fuzzer/src/compare/mod.rs b/tooling/ast_fuzzer/src/compare/mod.rs new file mode 100644 index 00000000000..f38386fed79 --- /dev/null +++ b/tooling/ast_fuzzer/src/compare/mod.rs @@ -0,0 +1,156 @@ +use acir::{FieldElement, native_types::WitnessStack}; +use acvm::pwg::{OpcodeResolutionError, ResolvedAssertionPayload}; +use arbitrary::{Arbitrary, Unstructured}; +use color_eyre::eyre::{self, WrapErr, bail}; +use nargo::{NargoError, errors::ExecutionError}; +use noirc_abi::{Abi, input_parser::InputValue}; +use noirc_evaluator::ssa::SsaEvaluatorOptions; +use noirc_frontend::monomorphization::ast::Program; + +mod compiled; +mod comptime; + +pub use compiled::{CompareArtifact, CompareCompiled, CompareMorph, ComparePipelines}; +pub use comptime::CompareComptime; + +#[derive(Clone, Debug, PartialEq)] +pub struct ExecOutput { + pub return_value: Option, + pub print_output: String, +} + +type ExecResult = (Result, NargoError>, String); + +/// Help iterate over the program(s) in the comparable artifact. +pub trait HasPrograms { + fn programs(&self) -> Vec<&Program>; +} + +/// Subset of [SsaEvaluatorOptions] that we want to vary. +/// +/// It exists to reduce noise in the printed results, compared to showing the full `SsaEvaluatorOptions`. +#[derive(Debug, Clone, Default)] +pub struct CompareOptions { + pub inliner_aggressiveness: i64, +} + +impl Arbitrary<'_> for CompareOptions { + fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { + Ok(Self { inliner_aggressiveness: *u.choose(&[i64::MIN, 0, i64::MAX])? }) + } +} + +impl CompareOptions { + /// Copy fields into an [SsaEvaluatorOptions] instance. + pub fn onto(&self, mut options: SsaEvaluatorOptions) -> SsaEvaluatorOptions { + options.inliner_aggressiveness = self.inliner_aggressiveness; + options + } +} + +/// Possible outcomes of the differential execution of two equivalent programs. +/// +/// Use [CompareResult::return_value_or_err] to do the final comparison between +/// the execution result. +pub enum CompareResult { + BothFailed(NargoError, NargoError), + LeftFailed(NargoError, ExecOutput), + RightFailed(ExecOutput, NargoError), + BothPassed(ExecOutput, ExecOutput), +} + +impl CompareResult { + fn new( + abi: &Abi, + (res1, print1): ExecResult, + (res2, print2): ExecResult, + ) -> eyre::Result { + let decode = |ws: WitnessStack| -> eyre::Result> { + let wm = &ws.peek().expect("there should be a main witness").witness; + let (_, r) = abi.decode(wm).wrap_err("abi::decode")?; + Ok(r) + }; + + match (res1, res2) { + (Err(e1), Err(e2)) => Ok(CompareResult::BothFailed(e1, e2)), + (Err(e1), Ok(ws2)) => Ok(CompareResult::LeftFailed( + e1, + ExecOutput { return_value: decode(ws2)?, print_output: print2 }, + )), + (Ok(ws1), Err(e2)) => Ok(CompareResult::RightFailed( + ExecOutput { return_value: decode(ws1)?, print_output: print1 }, + e2, + )), + (Ok(ws1), Ok(ws2)) => { + let o1 = ExecOutput { return_value: decode(ws1)?, print_output: print1 }; + let o2 = ExecOutput { return_value: decode(ws2)?, print_output: print2 }; + Ok(CompareResult::BothPassed(o1, o2)) + } + } + } + + /// Check that the programs agree on a return value. + /// + /// Returns an error if anything is different. + pub fn return_value_or_err(&self) -> eyre::Result> { + match self { + CompareResult::BothFailed(e1, e2) => { + if Self::errors_match(e1, e2) { + // Both programs failed the same way. + Ok(None) + } else { + bail!("both programs failed: {e1} vs {e2}\n{e1:?}\n{e2:?}") + } + } + CompareResult::LeftFailed(e, _) => { + bail!("first program failed: {e}\n{e:?}") + } + CompareResult::RightFailed(_, e) => { + bail!("second program failed: {e}\n{e:?}") + } + CompareResult::BothPassed(o1, o2) => { + if o1.return_value != o2.return_value { + bail!( + "programs disagree on return value:\n{:?}\n!=\n{:?}", + o1.return_value, + o2.return_value + ) + } else if o1.print_output != o2.print_output { + bail!( + "programs disagree on printed output:\n---\n{}\n\n---\n{}\n", + o1.print_output, + o2.print_output + ) + } else { + Ok(o1.return_value.as_ref()) + } + } + } + } + + /// Check whether two errors can be considered equivalent. + fn errors_match(e1: &NargoError, e2: &NargoError) -> bool { + use ExecutionError::*; + + // For now consider non-execution errors as failures we need to investigate. + let NargoError::ExecutionError(ee1) = e1 else { + return false; + }; + let NargoError::ExecutionError(ee2) = e2 else { + return false; + }; + + match (ee1, ee2) { + (AssertionFailed(p1, _, _), AssertionFailed(p2, _, _)) => p1 == p2, + (SolvingError(s1, _), SolvingError(s2, _)) => format!("{s1}") == format!("{s2}"), + (SolvingError(s, _), AssertionFailed(p, _, _)) + | (AssertionFailed(p, _, _), SolvingError(s, _)) => match (s, p) { + ( + OpcodeResolutionError::UnsatisfiedConstrain { .. }, + ResolvedAssertionPayload::String(s), + ) => s == "Attempted to divide by zero", + _ => false, + }, + } + } +}