diff --git a/.gitignore b/.gitignore index ace5be030ae..3987d09d2ca 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ pkg/ # Noir.js tooling/noir_js/lib +# fuzzing inputs (generated from 'test_programs') +tooling/nargo_fuzz_target/draft_inputs + # Nargo output *.proof *.acir diff --git a/Cargo.lock b/Cargo.lock index 77681fa9bdd..c7ec55f08be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,6 +149,18 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "afl" +version = "0.15.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda81c043843d2eeb489c3f30774953326fa043b7a6470d4c2ad7c3cdfd9847b" +dependencies = [ + "home", + "libc", + "rustc_version", + "xdg", +] + [[package]] name = "ahash" version = "0.8.11" @@ -2217,6 +2229,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "http" version = "1.3.1" @@ -3271,6 +3292,16 @@ dependencies = [ "toml", ] +[[package]] +name = "nargo_fuzz_target" +version = "1.0.0-beta.4" +dependencies = [ + "afl", + "nargo", + "nargo_cli", + "noirc_driver", +] + [[package]] name = "nargo_toml" version = "1.0.0-beta.4" @@ -6701,6 +6732,12 @@ dependencies = [ "tap", ] +[[package]] +name = "xdg" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" + [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index 50838703fc8..a5f644a024c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "tooling/nargo_fmt", "tooling/nargo_cli", "tooling/nargo_toml", + "tooling/nargo_fuzz_target", "tooling/noirc_artifacts", "tooling/noirc_artifacts_info", "tooling/noirc_abi", @@ -92,6 +93,7 @@ noirc_printable_type = { path = "compiler/noirc_printable_type" } # Noir tooling workspace dependencies noir_greybox_fuzzer = { path = "tooling/greybox_fuzzer" } nargo = { path = "tooling/nargo" } +nargo_cli = { path = "tooling/nargo_cli" } nargo_fmt = { path = "tooling/nargo_fmt" } nargo_toml = { path = "tooling/nargo_toml" } noir_lsp = { path = "tooling/lsp" } diff --git a/test_programs/README.md b/test_programs/README.md index f7e6cdc85dd..268e07a6edd 100644 --- a/test_programs/README.md +++ b/test_programs/README.md @@ -9,6 +9,10 @@ Integration tests for the Noir compiler are broken down into the following direc - `execution_panic`: a temporary location for tests that panic, but may or may not be valid or satisfiable Noir code. These tests should: + Each have a corresponding open issue + Be moved into another test folder once the panic is addressed +- `execution_stack_overflow`: a temporary location for tests that stack overflow, but may or may not be valid or satisfiable Noir code. These tests should: + + Each have a corresponding open issue + + Be moved into another test folder once the stack overflow is addressed + + NOTE: these tests are not currently run, either locally or CI The current testing flow can be thought of as shown: ```mermaid diff --git a/test_programs/execution_panic/invalid_attribute_span/Nargo.toml b/test_programs/execution_panic/invalid_attribute_span/Nargo.toml new file mode 100644 index 00000000000..2f80ac3c927 --- /dev/null +++ b/test_programs/execution_panic/invalid_attribute_span/Nargo.toml @@ -0,0 +1,6 @@ +[package] +name = "invalid_attribute_span" +type = "bin" +authors = [""] + +[dependencies] \ No newline at end of file diff --git a/test_programs/execution_panic/invalid_attribute_span/src/main.nr b/test_programs/execution_panic/invalid_attribute_span/src/main.nr new file mode 100644 index 00000000000..224043c74e4 Binary files /dev/null and b/test_programs/execution_panic/invalid_attribute_span/src/main.nr differ diff --git a/test_programs/execution_panic/invalid_ident_unwrap/Nargo.toml b/test_programs/execution_panic/invalid_ident_unwrap/Nargo.toml new file mode 100644 index 00000000000..f29ecbd0ff7 --- /dev/null +++ b/test_programs/execution_panic/invalid_ident_unwrap/Nargo.toml @@ -0,0 +1,6 @@ +[package] +name = "invalid_ident_unwrap" +type = "bin" +authors = [""] + +[dependencies] \ No newline at end of file diff --git a/test_programs/execution_panic/invalid_ident_unwrap/src/main.nr b/test_programs/execution_panic/invalid_ident_unwrap/src/main.nr new file mode 100644 index 00000000000..76ee6561376 --- /dev/null +++ b/test_programs/execution_panic/invalid_ident_unwrap/src/main.nr @@ -0,0 +1,9 @@ + + #[make_bar] + pubunfn foo((((((((((((&(((((((((((((((((((((((((((((((()(((fn main() { + round::<3, = 2 + // 3/2*2 2>([1, 2, 2, 3]; + ]); +} + +fn rou \ No newline at end of file diff --git a/test_programs/execution_stack_overflow/bracket_stack_overflow/Nargo.toml b/test_programs/execution_stack_overflow/bracket_stack_overflow/Nargo.toml new file mode 100644 index 00000000000..3254f54600d --- /dev/null +++ b/test_programs/execution_stack_overflow/bracket_stack_overflow/Nargo.toml @@ -0,0 +1,6 @@ +[package] +name = "bracket_stack_overflow" +type = "bin" +authors = [""] + +[dependencies] \ No newline at end of file diff --git a/test_programs/execution_stack_overflow/bracket_stack_overflow/src/main.nr b/test_programs/execution_stack_overflow/bracket_stack_overflow/src/main.nr new file mode 100644 index 00000000000..688bee69b94 --- /dev/null +++ b/test_programs/execution_stack_overflow/bracket_stack_overflow/src/main.nr @@ -0,0 +1,2 @@ +// TODO(https://github.com/noir-lang/noir/issues/8354): stack overflow +#[oracle([[[t[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ assert_eq(st.methodfn fo diff --git a/test_programs/execution_stack_overflow/parens_stack_overflow/Nargo.toml b/test_programs/execution_stack_overflow/parens_stack_overflow/Nargo.toml new file mode 100644 index 00000000000..e51a6d7a7a4 --- /dev/null +++ b/test_programs/execution_stack_overflow/parens_stack_overflow/Nargo.toml @@ -0,0 +1,6 @@ +[package] +name = "parens_stack_overflow" +type = "bin" +authors = [""] + +[dependencies] \ No newline at end of file diff --git a/test_programs/execution_stack_overflow/parens_stack_overflow/src/main.nr b/test_programs/execution_stack_overflow/parens_stack_overflow/src/main.nr new file mode 100644 index 00000000000..ca85cf8228b --- /dev/null +++ b/test_programs/execution_stack_overflow/parens_stack_overflow/src/main.nr @@ -0,0 +1,3 @@ +// TODO(https://github.com/noir-lang/noir/issues/8354): stack overflow +fn tr(n my_eqrSelf) -> bool { + ain(# {'  !Foo:: bool { + tr/fn main(mut x: i8, mut y: i8, z: i8) {{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{ + let mut s1: i8 = 5{{{{{{{{{{{{{ + let mut s1: i8 = 5; + let F: Fie let {{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{s i8 ; + let F: Fie let {{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{s i8 {{{{fn main(x: Field) { + diff --git a/tooling/nargo_cli/Cargo.toml b/tooling/nargo_cli/Cargo.toml index 11e9bbcc755..b8e74c77a94 100644 --- a/tooling/nargo_cli/Cargo.toml +++ b/tooling/nargo_cli/Cargo.toml @@ -13,6 +13,10 @@ workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "nargo_cli" +path = "src/lib.rs" + # Rename binary from `nargo_cli` to `nargo` [[bin]] name = "nargo" diff --git a/tooling/nargo_cli/src/cli/compile_cmd.rs b/tooling/nargo_cli/src/cli/compile_cmd.rs index 6764ad6773f..72fe467805d 100644 --- a/tooling/nargo_cli/src/cli/compile_cmd.rs +++ b/tooling/nargo_cli/src/cli/compile_cmd.rs @@ -28,7 +28,7 @@ use rayon::prelude::*; /// Compile the program and its secret execution trace into ACIR format #[derive(Debug, Clone, Args)] -pub(crate) struct CompileCommand { +pub struct CompileCommand { #[clap(flatten)] pub(super) package_options: PackageOptions, @@ -58,7 +58,8 @@ pub(crate) fn run(args: CompileCommand, workspace: Workspace) -> Result<(), CliE watch_workspace(&workspace, &args.compile_options) .map_err(|err| CliError::Generic(err.to_string()))?; } else { - compile_workspace_full(&workspace, &args.compile_options)?; + let debug_compile_stdin = None; + compile_workspace_full(&workspace, &args.compile_options, debug_compile_stdin)?; } Ok(()) } @@ -77,7 +78,8 @@ fn watch_workspace(workspace: &Workspace, compile_options: &CompileOptions) -> n let mut screen = std::io::stdout(); write!(screen, "{}", termion::cursor::Save).unwrap(); screen.flush().unwrap(); - let _ = compile_workspace_full(workspace, compile_options); + let debug_compile_stdin = None; + let _ = compile_workspace_full(workspace, compile_options, debug_compile_stdin); for res in rx { let debounced_events = res.map_err(|mut err| err.remove(0))?; @@ -98,7 +100,8 @@ fn watch_workspace(workspace: &Workspace, compile_options: &CompileOptions) -> n if noir_files_modified { write!(screen, "{}{}", termion::cursor::Restore, termion::clear::AfterCursor).unwrap(); screen.flush().unwrap(); - let _ = compile_workspace_full(workspace, compile_options); + let debug_compile_stdin = None; + let _ = compile_workspace_full(workspace, compile_options, debug_compile_stdin); } } @@ -108,14 +111,13 @@ fn watch_workspace(workspace: &Workspace, compile_options: &CompileOptions) -> n } /// Parse all files in the workspace. -fn parse_workspace(workspace: &Workspace, debug_compile_stdin: bool) -> (FileManager, ParsedFiles) { +fn parse_workspace( + workspace: &Workspace, + debug_compile_stdin: Option, +) -> (FileManager, ParsedFiles) { let mut file_manager = workspace.new_file_manager(); - if debug_compile_stdin { - let mut main_nr = String::new(); - let stdin = std::io::stdin(); - let mut stdin_handle = stdin.lock(); - stdin_handle.read_to_string(&mut main_nr).expect("reading from stdin to succeed"); + if let Some(main_nr) = debug_compile_stdin { file_manager.add_file_with_source(Path::new("src/main.nr"), main_nr); } else { insert_all_files_for_workspace_into_file_manager(workspace, &mut file_manager); @@ -127,12 +129,20 @@ fn parse_workspace(workspace: &Workspace, debug_compile_stdin: bool) -> (FileMan /// Parse and compile the entire workspace, then report errors. /// This is the main entry point used by all other commands that need compilation. -pub(super) fn compile_workspace_full( +pub fn compile_workspace_full( workspace: &Workspace, compile_options: &CompileOptions, + debug_compile_stdin: Option, // use this String as STDIN if present ) -> Result<(), CliError> { - let (workspace_file_manager, parsed_files) = - parse_workspace(workspace, compile_options.debug_compile_stdin); + let mut debug_compile_stdin = debug_compile_stdin; + if compile_options.debug_compile_stdin && debug_compile_stdin.is_none() { + let mut main_nr = String::new(); + let stdin = std::io::stdin(); + let mut stdin_handle = stdin.lock(); + stdin_handle.read_to_string(&mut main_nr).expect("reading from stdin to succeed"); + debug_compile_stdin = Some(main_nr); + } + let (workspace_file_manager, parsed_files) = parse_workspace(workspace, debug_compile_stdin); let compiled_workspace = compile_workspace(&workspace_file_manager, &parsed_files, workspace, compile_options); @@ -400,7 +410,7 @@ mod tests { // This could be `.par_iter()` but then error messages are no longer reported test_workspaces.iter().for_each(|workspace| { - let debug_compile_stdin = false; + let debug_compile_stdin = None; let (file_manager, parsed_files) = parse_workspace(workspace, debug_compile_stdin); let binary_packages = workspace.into_iter().filter(|package| package.is_binary()); diff --git a/tooling/nargo_cli/src/cli/execute_cmd.rs b/tooling/nargo_cli/src/cli/execute_cmd.rs index 7ba95a16eeb..5b1baaaf36e 100644 --- a/tooling/nargo_cli/src/cli/execute_cmd.rs +++ b/tooling/nargo_cli/src/cli/execute_cmd.rs @@ -52,7 +52,8 @@ impl WorkspaceCommand for ExecuteCommand { pub(crate) fn run(args: ExecuteCommand, workspace: Workspace) -> Result<(), CliError> { // Compile the full workspace in order to generate any build artifacts. - compile_workspace_full(&workspace, &args.compile_options)?; + let debug_compile_stdin = None; + compile_workspace_full(&workspace, &args.compile_options, debug_compile_stdin)?; let binary_packages = workspace.into_iter().filter(|package| package.is_binary()); for package in binary_packages { diff --git a/tooling/nargo_cli/src/cli/info_cmd.rs b/tooling/nargo_cli/src/cli/info_cmd.rs index c8a9c289fc0..117e5359737 100644 --- a/tooling/nargo_cli/src/cli/info_cmd.rs +++ b/tooling/nargo_cli/src/cli/info_cmd.rs @@ -67,7 +67,8 @@ pub(crate) fn run(mut args: InfoCommand, workspace: Workspace) -> Result<(), Cli args.compile_options.force_brillig = true; } // Compile the full workspace in order to generate any build artifacts. - compile_workspace_full(&workspace, &args.compile_options)?; + let debug_compile_stdin = None; + compile_workspace_full(&workspace, &args.compile_options, debug_compile_stdin)?; let binary_packages: Vec<(Package, ProgramArtifact)> = workspace .into_iter() diff --git a/tooling/nargo_cli/src/cli/mod.rs b/tooling/nargo_cli/src/cli/mod.rs index 958472bb463..05a47eba22c 100644 --- a/tooling/nargo_cli/src/cli/mod.rs +++ b/tooling/nargo_cli/src/cli/mod.rs @@ -18,7 +18,7 @@ use color_eyre::eyre; use crate::errors::CliError; mod check_cmd; -mod compile_cmd; +pub mod compile_cmd; mod dap_cmd; mod debug_cmd; mod execute_cmd; @@ -57,7 +57,7 @@ struct NargoCli { #[non_exhaustive] #[derive(Args, Clone, Debug)] -pub(crate) struct NargoConfig { +pub struct NargoConfig { // REMINDER: Also change this flag in the LSP test lens if renamed #[arg(long, hide = true, global = true, default_value = "./", value_parser = parse_path)] program_dir: PathBuf, diff --git a/tooling/nargo_cli/src/errors.rs b/tooling/nargo_cli/src/errors.rs index 25751c11270..402fc6d8559 100644 --- a/tooling/nargo_cli/src/errors.rs +++ b/tooling/nargo_cli/src/errors.rs @@ -7,7 +7,7 @@ use std::path::PathBuf; use thiserror::Error; #[derive(Debug, Error)] -pub(crate) enum CliError { +pub enum CliError { #[error("{0}")] Generic(String), diff --git a/tooling/nargo_cli/src/lib.rs b/tooling/nargo_cli/src/lib.rs new file mode 100644 index 00000000000..1fcdb979006 --- /dev/null +++ b/tooling/nargo_cli/src/lib.rs @@ -0,0 +1,5 @@ +// NOTE: this code should all be used in the 'bin' target +#![allow(dead_code)] + +pub mod cli; +pub mod errors; diff --git a/tooling/nargo_cli/src/main.rs b/tooling/nargo_cli/src/main.rs index 33e18ce6f94..938ea250425 100644 --- a/tooling/nargo_cli/src/main.rs +++ b/tooling/nargo_cli/src/main.rs @@ -7,7 +7,7 @@ //! This name was used because it sounds like `cargo` and //! Noir Package Manager abbreviated is npm, which is already taken. -mod cli; +pub mod cli; mod errors; use std::env; @@ -19,6 +19,8 @@ use tracing_subscriber::{EnvFilter, fmt::format::FmtSpan}; // TODO: Currently only used by benches. use noir_artifact_cli as _; +// NOTE: Currently only used for fuzzing. +use nargo_cli as _; const PANIC_MESSAGE: &str = "This is a bug. We may have already fixed this in newer versions of Nargo so try searching for similar issues at https://github.com/noir-lang/noir/issues/.\nIf there isn't an open issue for this bug, consider opening one at https://github.com/noir-lang/noir/issues/new?labels=bug&template=bug_report.yml"; diff --git a/tooling/nargo_fuzz_target/Cargo.toml b/tooling/nargo_fuzz_target/Cargo.toml new file mode 100644 index 00000000000..3c6639f16bf --- /dev/null +++ b/tooling/nargo_fuzz_target/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "nargo_fuzz_target" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +nargo.workspace = true +nargo_cli.workspace = true +noirc_driver.workspace = true +afl = "*" diff --git a/tooling/nargo_fuzz_target/README.md b/tooling/nargo_fuzz_target/README.md new file mode 100644 index 00000000000..15aba146d97 --- /dev/null +++ b/tooling/nargo_fuzz_target/README.md @@ -0,0 +1,211 @@ +# `nargo_fuzz_target` + +To set up the inputs, empty the `draft_inputs` directory, if present, and run: + +```bash +./populate_inputs.rb +``` + +To copy all `.nr` files with up to `512` bytes to the `draft_inputs` folder. + +Then, copy any relevant inputs (e.g. all of them) to `inputs` and run with: + +```bash +# to install 'cargo-afl' +cargo install cargo-afl + +cargo afl build +cargo afl fuzz -i inputs -o outputs ../../target/debug/nargo_fuzz_target +``` + +## Collecting results: + +```bash +# (from this directory) +./collect_unique_crashes.rb +``` + +To only collect outputs and deduplicate more manually, run: + +```ruby +# ❯ ./collect_outputs.rb ./collected_outputs + +def ff(x) + `bash -c "cat #{x} | ../../../target/release/nargo compile --debug-compile-stdin 2>&1"` +end + +def gg(x) + puts('-'*80) + puts ff(x) + puts('-'*80) +end + +# from ./collected_outputs/ +Dir['./*'].map{|x| ff(x)}.sort.uniq.each{|x|puts('-'*80);puts(x);puts('-'*80)} +``` + +## Ramdisk + +Using a ramdisk for the `inputs` and `outputs` folder may improve performance: + +```bash +# make dir +❯ sudo mkdir /tmp/nargo_fuzz_ramdisk + +# make accessible to other users +❯ sudo chmod 777 /tmp/nargo_fuzz_ramdisk + +# now, the nargo binary is ~400MB +❯ ls -lah ../../target/debug/nargo_fuzz_target +-rwxrwxr-x 2 michael michael 398M May 1 11:32 ../../target/debug/nargo_fuzz_target + +# so we round up to the next power of 2: +❯ sudo mount -t tmpfs -o size=512m nargo_fuzz_ramdisk /tmp/nargo_fuzz_ramdisk + +# check for successful mount +❯ mount | tail -n 1 +nargo_fuzz_ramdisk on /tmp/nargo_fuzz_ramdisk type tmpfs (rw,relatime,size=524288k,inode64) + +# copy inputs/outputs to the ramdisk +❯ .. + +# CLEANUP +# delete the ramdisk +sudo umount /tmp/nargo_fuzz_ramdisk/ +``` + + +## Example Results + +Simple run in debug mode: + +``` + AFL ++4.21c {default} (../../target/debug/nargo_fuzz_target) [explore] +┌─ process timing ────────────────────────────────────┬─ overall results ────┐ +│ run time : 0 days, 7 hrs, 54 min, 6 sec │ cycles done : 0 │ +│ last new find : 0 days, 0 hrs, 0 min, 55 sec │ corpus count : 3361 │ +│last saved crash : 0 days, 0 hrs, 6 min, 33 sec │saved crashes : 30 │ +│ last saved hang : none seen yet │ saved hangs : 0 │ +├─ cycle progress ─────────────────────┬─ map coverage┴──────────────────────┤ +│ now processing : 1980*1 (58.9%) │ map density : 0.84% / 1.14% │ +│ runs timed out : 0 (0.00%) │ count coverage : 2.84 bits/tuple │ +├─ stage progress ─────────────────────┼─ findings in depth ─────────────────┤ +│ now trying : havoc │ favored items : 267 (7.94%) │ +│ stage execs : 13/80 (16.25%) │ new edges on : 483 (14.37%) │ +│ total execs : 734k │ total crashes : 34 (30 saved) │ +│ exec speed : 20.23/sec (slow!) │ total tmouts : 0 (0 saved) │ +├─ fuzzing strategy yields ────────────┴─────────────┬─ item geometry ───────┤ +│ bit flips : 3/7336, 3/7333, 2/7327 │ levels : 36 │ +│ byte flips : 0/917, 0/914, 0/908 │ pending : 2487 │ +│ arithmetics : 9/64.1k, 0/127k, 0/126k │ pend fav : 0 │ +│ known ints : 0/8235, 1/34.7k, 2/50.8k │ own finds : 3352 │ +│ dictionary : 0/0, 0/0, 0/0, 0/0 │ imported : 0 │ +│havoc/splice : 1145/163k, 1917/265k │ stability : 95.95% │ +│py/custom/rq : unused, unused, 282/175k, 0/0 ├───────────────────────┘ +│ trim/eff : 0.18%/75.8k, 99.35% │ [cpu000: 18%] +└─ strategy: explore ────────── state: in progress ──┘^C + ++++ Testing aborted by user +++ + +[!] Stopped during the first cycle, results may be incomplete. +``` + +Simple run in release mode: + +``` + AFL ++4.21c {default} (../../target/release/nargo_fuzz_target) [explore] +┌─ process timing ────────────────────────────────────┬─ overall results ────┐ +│ run time : 0 days, 7 hrs, 49 min, 31 sec │ cycles done : 0 │ +│ last new find : 0 days, 0 hrs, 0 min, 4 sec │ corpus count : 3089 │ +│last saved crash : 0 days, 0 hrs, 2 min, 52 sec │saved crashes : 70 │ +│ last saved hang : none seen yet │ saved hangs : 0 │ +├─ cycle progress ─────────────────────┬─ map coverage┴──────────────────────┤ +│ now processing : 3068.1 (99.3%) │ map density : 0.87% / 1.17% │ +│ runs timed out : 1 (0.03%) │ count coverage : 2.66 bits/tuple │ +├─ stage progress ─────────────────────┼─ findings in depth ─────────────────┤ +│ now trying : trim 16/16 │ favored items : 289 (9.36%) │ +│ stage execs : 2/17 (11.76%) │ new edges on : 538 (17.42%) │ +│ total execs : 684k │ total crashes : 87 (70 saved) │ +│ exec speed : 29.48/sec (slow!) │ total tmouts : 0 (0 saved) │ +├─ fuzzing strategy yields ────────────┴─────────────┬─ item geometry ───────┤ +│ bit flips : 11/8688, 2/8684, 1/8676 │ levels : 23 │ +│ byte flips : 0/1086, 2/1082, 0/1074 │ pending : 2402 │ +│ arithmetics : 16/75.8k, 0/149k, 0/148k │ pend fav : 7 │ +│ known ints : 0/9728, 4/41.0k, 2/60.0k │ own finds : 3080 │ +│ dictionary : 0/0, 0/0, 0/0, 0/0 │ imported : 0 │ +│havoc/splice : 719/116k, 2052/218k │ stability : 96.04% │ +│py/custom/rq : unused, unused, 320/222k, 0/0 ├───────────────────────┘ +│ trim/eff : 0.01%/77.4k, 98.80% │ [cpu001: 20%] +└─ strategy: explore ────────── state: in progress ──┘^C + ++++ Testing aborted by user +++ + +[!] Stopped during the first cycle, results may be incomplete. +``` + +Simple run in release mode (ramdisk): + +``` + AFL ++4.21c {default} (/tmp/nargo_fuzz_ramdisk/nargo_fuzz_target) [explore] +┌─ process timing ────────────────────────────────────┬─ overall results ────┐ +│ run time : 0 days, 1 hrs, 2 min, 55 sec │ cycles done : 0 │ +│ last new find : 0 days, 0 hrs, 0 min, 1 sec │ corpus count : 1350 │ +│last saved crash : none seen yet │saved crashes : 0 │ +│ last saved hang : none seen yet │ saved hangs : 0 │ +├─ cycle progress ─────────────────────┬─ map coverage┴──────────────────────┤ +│ now processing : 27.1 (2.0%) │ map density : 0.83% / 1.04% │ +│ runs timed out : 0 (0.00%) │ count coverage : 2.15 bits/tuple │ +├─ stage progress ─────────────────────┼─ findings in depth ─────────────────┤ +│ now trying : splice 8 │ favored items : 157 (11.63%) │ +│ stage execs : 773/1536 (50.33%) │ new edges on : 257 (19.04%) │ +│ total execs : 100k │ total crashes : 0 (0 saved) │ +│ exec speed : 29.59/sec (slow!) │ total tmouts : 0 (0 saved) │ +├─ fuzzing strategy yields ────────────┴─────────────┬─ item geometry ───────┤ +│ bit flips : 7/144, 2/143, 2/141 │ levels : 10 │ +│ byte flips : 0/18, 1/17, 0/15 │ pending : 1298 │ +│ arithmetics : 14/1189, 0/1707, 0/1540 │ pend fav : 123 │ +│ known ints : 0/142, 3/595, 2/800 │ own finds : 1341 │ +│ dictionary : 0/0, 0/0, 0/0, 0/0 │ imported : 0 │ +│havoc/splice : 129/7660, 1097/54.0k │ stability : 95.57% │ +│py/custom/rq : unused, unused, 51/16.0k, 0/0 ├───────────────────────┘ +│ trim/eff : 0.11%/3471, 72.22% │ [cpu002: 15%] +└─ strategy: explore ────────── state: in progress ──┘^C + ++++ Testing aborted by user +++ + +[!] Stopped during the first cycle, results may be incomplete. +``` + +Independent simple release mode run before setting up proper parallel run: + +``` + AFL ++4.21c {fuzzer3} (../../target/release/nargo_fuzz_target) [explore] +┌─ process timing ────────────────────────────────────┬─ overall results ────┐ +│ run time : 0 days, 0 hrs, 54 min, 48 sec │ cycles done : 0 │ +│ last new find : 0 days, 0 hrs, 0 min, 23 sec │ corpus count : 1788 │ +│last saved crash : none seen yet │saved crashes : 0 │ +│ last saved hang : none seen yet │ saved hangs : 0 │ +├─ cycle progress ─────────────────────┬─ map coverage┴──────────────────────┤ +│ now processing : 1637.1 (91.6%) │ map density : 0.85% / 1.06% │ +│ runs timed out : 0 (0.00%) │ count coverage : 2.27 bits/tuple │ +├─ stage progress ─────────────────────┼─ findings in depth ─────────────────┤ +│ now trying : colorization │ favored items : 206 (11.52%) │ +│ stage execs : 496/500 (99.20%) │ new edges on : 329 (18.40%) │ +│ total execs : 84.7k │ total crashes : 0 (0 saved) │ +│ exec speed : 19.07/sec (zzzz...) │ total tmouts : 0 (0 saved) │ +├─ fuzzing strategy yields ────────────┴─────────────┬─ item geometry ───────┤ +│ bit flips : 0/0, 0/0, 0/0 │ levels : 4 │ +│ byte flips : 0/0, 0/0, 0/0 │ pending : 1754 │ +│ arithmetics : 0/0, 0/0, 0/0 │ pend fav : 185 │ +│ known ints : 0/0, 0/0, 0/0 │ own finds : 1280 │ +│ dictionary : 0/0, 0/0, 0/0, 0/0 │ imported : 499 │ +│havoc/splice : 101/2064, 1135/50.8k │ stability : 95.62% │ +│py/custom/rq : unused, unused, 36/5243, 0/0 ├───────────────────────┘ +│ trim/eff : 1.09%/1079, n/a │ [cpu005: 48%] +└─ strategy: explore ────────── state: in progress ──┘^C + ++++ Testing aborted by user +++ + +[!] Stopped during the first cycle, results may be incomplete. +``` + diff --git a/tooling/nargo_fuzz_target/collect_outputs.rb b/tooling/nargo_fuzz_target/collect_outputs.rb new file mode 100755 index 00000000000..0e44302214e --- /dev/null +++ b/tooling/nargo_fuzz_target/collect_outputs.rb @@ -0,0 +1,34 @@ +#!/usr/bin/env ruby + +require 'fileutils' + +output_dirs = [ + "outputs", + "outputs-big", + "outputs-parallel", + "outputs-ramdisk", + "outputs-release", +] + +collected_output_dir = ARGV[0] +if collected_output_dir.nil? + raise "the first argument is required: [dir-for-all-outputs]" +end + +num_crashes = 0 +puts "copying crashing inputs to #{collected_output_dir}" + +# [output_dir]/[worker_dir]/crashes/id:[..] +output_dirs.each do |output_dir| + Dir["#{output_dir}/*"].filter do |worker_output_dir| + if Dir.exist? worker_output_dir + Dir["#{worker_output_dir}/crashes/id:*"].each do |crash_path| + puts "copying #{crash_path}" + FileUtils.cp crash_path, ARGV[0] + num_crashes += 1 + end + end + end +end + +puts "copied #{num_crashes} crashes." diff --git a/tooling/nargo_fuzz_target/collect_unique_crashes.rb b/tooling/nargo_fuzz_target/collect_unique_crashes.rb new file mode 100755 index 00000000000..51dc37226a0 --- /dev/null +++ b/tooling/nargo_fuzz_target/collect_unique_crashes.rb @@ -0,0 +1,51 @@ +#!/usr/bin/env ruby + +require 'fileutils' +require 'open3' + +def run_nargo(program_path, program_str) + nargo_path = "../../target/release/nargo" + cmd_str = "#{nargo_path} compile --debug-compile-stdin --pedantic-solving" + + # If opts[:stdin_data] is specified, it is sent to the command’s standard input. + stdout_and_stderr_str, status = Open3.capture2e(cmd_str, stdin_data: program_str) + + if status.success? + raise "unexpected successful input: #{program_path}" + end + + stdout_and_stderr_str +end + +# update ./collected_outputs +system "./collect_outputs.rb ./collected_outputs" +puts + +num_crashes = 0 +Dir['./collected_outputs/*'].map do |collected_output_path| + [collected_output_path, File.read(collected_output_path)] +end.map do |collected_output_path, collected_output_str| + puts "running 'nargo' on #{collected_output_path}" + nargo_output = run_nargo collected_output_path, collected_output_str + [collected_output_path, collected_output_str, nargo_output] +end.group_by do |collected_output_path, collected_output_str, nargo_output| + nargo_output +end.map do |nargo_output, collected_outputs| + smallest_input_path_and_str = collected_outputs.min_by do |_, collected_output_str, _| + collected_output_str.length + end + puts "#{collected_outputs.length} non-unique crashes" + smallest_input_path = smallest_input_path_and_str[0] + smallest_input_str = smallest_input_path_and_str[1] + + puts "copying #{smallest_input_path.inspect}, (input size: #{smallest_input_str.length})" + FileUtils.cp smallest_input_path, './unique_crashes' + + smallest_input_basename = File.basename smallest_input_path + File.write "./unique_crashes/#{smallest_input_basename}.out", nargo_output + + num_crashes += 1 +end +puts + +puts "copied #{num_crashes} output-unique crashes." diff --git a/tooling/nargo_fuzz_target/populate_inputs.rb b/tooling/nargo_fuzz_target/populate_inputs.rb new file mode 100755 index 00000000000..55664c9ed40 --- /dev/null +++ b/tooling/nargo_fuzz_target/populate_inputs.rb @@ -0,0 +1,31 @@ +#!/usr/bin/env ruby + +require 'fileutils' + +# NOTE: current files are pared down from this one-liner +# Dir['../../test_programs/compile_success_empty/**/*.nr'].select{|x|`stat -f%z #{x}`.chomp.to_i <= 120}.sort.join(' ') + +target_path = 'test_programs' +# target_path = 'noir_stdlib' + +puts "Copying all #{target_path} to inputs (renamed).." +num_inputs = 0 + +# create target directory unless it already exists +unless Dir.exist? 'draft_inputs' + FileUtils.mkdir 'draft_inputs' +end + +Dir["../../#{target_path}/**/*.nr"].each do |path| + new_filename = path + .sub(/^\.\.\/\.\.\/test_programs\//, '') + .gsub(/\//, "__") + + unless path.match?(/benchmark/) || File.read(path).length > 512 + puts path + FileUtils.cp path, "draft_inputs/#{new_filename}" + num_inputs += 1 + end +end + +puts "#{num_inputs} inputs collected" diff --git a/tooling/nargo_fuzz_target/src/main.rs b/tooling/nargo_fuzz_target/src/main.rs new file mode 100644 index 00000000000..1d6b4ca9696 --- /dev/null +++ b/tooling/nargo_fuzz_target/src/main.rs @@ -0,0 +1,39 @@ +#[macro_use] +extern crate afl; +extern crate nargo; +extern crate nargo_cli; +extern crate noirc_driver; + +use std::path::PathBuf; + +use nargo::workspace::Workspace; +use nargo_cli::cli::compile_cmd::compile_workspace_full; +use noirc_driver::CompileOptions; + +fn main() { + fuzz!(|data: &[u8]| { + if let Ok(debug_compile_stdin) = std::str::from_utf8(data) { + let workspace = Workspace { + root_dir: PathBuf::new(), + target_dir: None, + members: vec![], + selected_package_index: None, + + // used by LSP + is_assumed: false, + }; + + let compile_options = CompileOptions { + debug_compile_stdin: true, + pedantic_solving: true, + ..Default::default() + }; + + let _ = compile_workspace_full( + &workspace, + &compile_options, + Some(debug_compile_stdin.to_string()), + ); + } + }); +}