From 2fcc1465955df43f021a3d543c5f29eb6c4fef0f Mon Sep 17 00:00:00 2001 From: Tom French Date: Tue, 12 Sep 2023 18:02:04 +0100 Subject: [PATCH 1/7] chore: embed file map into `CompiledProgram` --- compiler/noirc_driver/Cargo.toml | 2 +- compiler/noirc_driver/src/debug.rs | 45 +++++++++++++++++++ compiler/noirc_driver/src/lib.rs | 7 ++- compiler/noirc_driver/src/program.rs | 6 +++ tooling/nargo/src/artifacts/debug.rs | 10 +---- .../nargo_cli/src/cli/codegen_verifier_cmd.rs | 2 +- tooling/nargo_cli/src/cli/compile_cmd.rs | 38 ++++++++-------- tooling/nargo_cli/src/cli/execute_cmd.rs | 14 +++--- tooling/nargo_cli/src/cli/info_cmd.rs | 2 +- tooling/nargo_cli/src/cli/prove_cmd.rs | 6 ++- tooling/nargo_cli/src/cli/verify_cmd.rs | 2 +- 11 files changed, 90 insertions(+), 44 deletions(-) create mode 100644 compiler/noirc_driver/src/debug.rs diff --git a/compiler/noirc_driver/Cargo.toml b/compiler/noirc_driver/Cargo.toml index 2afc7a4cb53..bd38371f2ad 100644 --- a/compiler/noirc_driver/Cargo.toml +++ b/compiler/noirc_driver/Cargo.toml @@ -15,4 +15,4 @@ noirc_abi.workspace = true acvm.workspace = true fm.workspace = true serde.workspace = true -base64.workspace = true \ No newline at end of file +base64.workspace = true diff --git a/compiler/noirc_driver/src/debug.rs b/compiler/noirc_driver/src/debug.rs new file mode 100644 index 00000000000..9808c9b54a2 --- /dev/null +++ b/compiler/noirc_driver/src/debug.rs @@ -0,0 +1,45 @@ +use fm::{FileId, FileManager}; +use noirc_errors::debug_info::DebugInfo; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::PathBuf, +}; + +/// For a given file, we store the source code and the path to the file +/// so consumers of the debug artifact can reconstruct the original source code structure. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DebugFile { + pub source: String, + pub path: PathBuf, +} + +pub(crate) fn filter_relevant_files( + debug_symbols: &[DebugInfo], + file_manager: &FileManager, +) -> BTreeMap { + let files_with_debug_symbols: BTreeSet = debug_symbols + .iter() + .flat_map(|function_symbols| { + function_symbols + .locations + .values() + .filter_map(|call_stack| call_stack.last().map(|location| location.file)) + }) + .collect(); + + let mut file_map = BTreeMap::new(); + + for file_id in files_with_debug_symbols { + let file_source = file_manager.fetch_file(file_id).source(); + + file_map.insert( + file_id, + DebugFile { + source: file_source.to_string(), + path: file_manager.path(file_id).to_path_buf(), + }, + ); + } + file_map +} diff --git a/compiler/noirc_driver/src/lib.rs b/compiler/noirc_driver/src/lib.rs index a608879ce77..1c8d1b19343 100644 --- a/compiler/noirc_driver/src/lib.rs +++ b/compiler/noirc_driver/src/lib.rs @@ -4,6 +4,7 @@ #![warn(clippy::semicolon_if_nothing_returned)] use clap::Args; +use debug::filter_relevant_files; use fm::FileId; use noirc_abi::{AbiParameter, AbiType}; use noirc_errors::{CustomDiagnostic, FileDiagnostic}; @@ -17,9 +18,11 @@ use serde::{Deserialize, Serialize}; use std::path::Path; mod contract; +mod debug; mod program; pub use contract::{CompiledContract, ContractFunction, ContractFunctionType}; +pub use debug::DebugFile; pub use program::CompiledProgram; const STD_CRATE_NAME: &str = "std"; @@ -269,5 +272,7 @@ pub fn compile_no_check( let (circuit, debug, abi) = create_circuit(context, program, options.show_ssa, options.show_brillig)?; - Ok(CompiledProgram { circuit, debug, abi }) + let file_map = filter_relevant_files(&[debug.clone()], &context.file_manager); + + Ok(CompiledProgram { circuit, debug, abi, file_map }) } diff --git a/compiler/noirc_driver/src/program.rs b/compiler/noirc_driver/src/program.rs index 9323f90d522..1ed2b0ddddc 100644 --- a/compiler/noirc_driver/src/program.rs +++ b/compiler/noirc_driver/src/program.rs @@ -1,15 +1,21 @@ +use std::collections::BTreeMap; + use acvm::acir::circuit::Circuit; +use fm::FileId; use base64::Engine; use noirc_errors::debug_info::DebugInfo; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use super::debug::DebugFile; + #[derive(Debug, Serialize, Deserialize, Clone)] pub struct CompiledProgram { #[serde(serialize_with = "serialize_circuit", deserialize_with = "deserialize_circuit")] pub circuit: Circuit, pub abi: noirc_abi::Abi, pub debug: DebugInfo, + pub file_map: BTreeMap, } pub(crate) fn serialize_circuit(circuit: &Circuit, s: S) -> Result diff --git a/tooling/nargo/src/artifacts/debug.rs b/tooling/nargo/src/artifacts/debug.rs index 2a201a82c48..3c173f34876 100644 --- a/tooling/nargo/src/artifacts/debug.rs +++ b/tooling/nargo/src/artifacts/debug.rs @@ -1,22 +1,14 @@ use codespan_reporting::files::{Error, Files, SimpleFile}; +use noirc_driver::DebugFile; use noirc_errors::debug_info::DebugInfo; use serde::{Deserialize, Serialize}; use std::{ collections::{BTreeMap, BTreeSet}, ops::Range, - path::PathBuf, }; use fm::{FileId, FileManager, PathString}; -/// For a given file, we store the source code and the path to the file -/// so consumers of the debug artifact can reconstruct the original source code structure. -#[derive(Debug, Serialize, Deserialize)] -pub struct DebugFile { - pub source: String, - pub path: PathBuf, -} - /// A Debug Artifact stores, for a given program, the debug info for every function /// along with a map of file Id to the source code so locations in debug info can be mapped to source code they point to. #[derive(Debug, Serialize, Deserialize)] diff --git a/tooling/nargo_cli/src/cli/codegen_verifier_cmd.rs b/tooling/nargo_cli/src/cli/codegen_verifier_cmd.rs index 6199bf0761d..16ff311f704 100644 --- a/tooling/nargo_cli/src/cli/codegen_verifier_cmd.rs +++ b/tooling/nargo_cli/src/cli/codegen_verifier_cmd.rs @@ -81,7 +81,7 @@ fn smart_contract_for_package( let preprocessed_program = if circuit_build_path.exists() { read_program_from_file(circuit_build_path)? } else { - let (program, _) = + let program = compile_bin_package(package, compile_options, np_language, &is_opcode_supported)?; PreprocessedProgram { diff --git a/tooling/nargo_cli/src/cli/compile_cmd.rs b/tooling/nargo_cli/src/cli/compile_cmd.rs index 61d3b64a47a..447bcdb6f20 100644 --- a/tooling/nargo_cli/src/cli/compile_cmd.rs +++ b/tooling/nargo_cli/src/cli/compile_cmd.rs @@ -72,13 +72,12 @@ pub(crate) fn run( .partition(|package| package.is_binary()); // Compile all of the packages in parallel. - let program_results: Vec<(FileManager, CompilationResult<(CompiledProgram, DebugArtifact)>)> = - binary_packages - .par_iter() - .map(|package| { - compile_program(package, &args.compile_options, np_language, &is_opcode_supported) - }) - .collect(); + let program_results: Vec<(FileManager, CompilationResult)> = binary_packages + .par_iter() + .map(|package| { + compile_program(package, &args.compile_options, np_language, &is_opcode_supported) + }) + .collect(); #[allow(clippy::type_complexity)] let contract_results: Vec<( FileManager, @@ -91,7 +90,7 @@ pub(crate) fn run( .collect(); // Report any warnings/errors which were encountered during compilation. - let compiled_programs: Vec<(CompiledProgram, DebugArtifact)> = program_results + let compiled_programs: Vec = program_results .into_iter() .map(|(file_manager, compilation_result)| { report_errors(compilation_result, &file_manager, args.compile_options.deny_warnings) @@ -105,8 +104,8 @@ pub(crate) fn run( .collect::>()?; // Save build artifacts to disk. - for (package, (program, debug_artifact)) in binary_packages.into_iter().zip(compiled_programs) { - save_program(debug_artifact, program, package, &circuit_dir, args.output_debug); + for (package, program) in binary_packages.into_iter().zip(compiled_programs) { + save_program(program, package, &circuit_dir, args.output_debug); } for (package, contracts_with_debug_artifacts) in contract_packages.into_iter().zip(compiled_contracts) @@ -122,7 +121,7 @@ pub(crate) fn compile_bin_package( compile_options: &CompileOptions, np_language: Language, is_opcode_supported: &impl Fn(&Opcode) -> bool, -) -> Result<(CompiledProgram, DebugArtifact), CliError> { +) -> Result { if package.is_library() { return Err(CompileError::LibraryCrate(package.name.clone()).into()); } @@ -130,10 +129,9 @@ pub(crate) fn compile_bin_package( let (file_manager, compilation_result) = compile_program(package, compile_options, np_language, &is_opcode_supported); - let (program, debug_artifact) = - report_errors(compilation_result, &file_manager, compile_options.deny_warnings)?; + let program = report_errors(compilation_result, &file_manager, compile_options.deny_warnings)?; - Ok((program, debug_artifact)) + Ok(program) } pub(crate) fn compile_contract_package( @@ -154,7 +152,7 @@ fn compile_program( compile_options: &CompileOptions, np_language: Language, is_opcode_supported: &impl Fn(&Opcode) -> bool, -) -> (FileManager, CompilationResult<(CompiledProgram, DebugArtifact)>) { +) -> (FileManager, CompilationResult) { let (mut context, crate_id) = prepare_package(package); let (program, warnings) = @@ -170,10 +168,7 @@ fn compile_program( nargo::ops::optimize_program(program, np_language, &is_opcode_supported) .expect("Backend does not support an opcode that is in the IR"); - let debug_artifact = - DebugArtifact::new(vec![optimized_program.debug.clone()], &context.file_manager); - - (context.file_manager, Ok(((optimized_program, debug_artifact), warnings))) + (context.file_manager, Ok((optimized_program, warnings))) } fn compile_contracts( @@ -207,7 +202,6 @@ fn compile_contracts( } fn save_program( - debug_artifact: DebugArtifact, program: CompiledProgram, package: &Package, circuit_dir: &Path, @@ -222,6 +216,10 @@ fn save_program( save_program_to_file(&preprocessed_program, &package.name, circuit_dir); if output_debug { + let debug_artifact = DebugArtifact { + debug_symbols: vec![program.debug.clone()], + file_map: program.file_map, + }; let circuit_name: String = (&package.name).into(); save_debug_artifact_to_file(&debug_artifact, &circuit_name, circuit_dir); } diff --git a/tooling/nargo_cli/src/cli/execute_cmd.rs b/tooling/nargo_cli/src/cli/execute_cmd.rs index a08cfb09995..8c434f8fe21 100644 --- a/tooling/nargo_cli/src/cli/execute_cmd.rs +++ b/tooling/nargo_cli/src/cli/execute_cmd.rs @@ -56,15 +56,11 @@ pub(crate) fn run( let (np_language, is_opcode_supported) = backend.get_backend_info()?; for package in &workspace { - let (compiled_program, debug_artifact) = + let compiled_program = compile_bin_package(package, &args.compile_options, np_language, &is_opcode_supported)?; - let (return_value, solved_witness) = execute_program_and_decode( - compiled_program, - debug_artifact, - package, - &args.prover_name, - )?; + let (return_value, solved_witness) = + execute_program_and_decode(compiled_program, package, &args.prover_name)?; println!("[{}] Circuit witness successfully solved", package.name); if let Some(return_value) = return_value { @@ -81,11 +77,11 @@ pub(crate) fn run( fn execute_program_and_decode( program: CompiledProgram, - debug_artifact: DebugArtifact, package: &Package, prover_name: &str, ) -> Result<(Option, WitnessMap), CliError> { - let CompiledProgram { abi, circuit, .. } = program; + let CompiledProgram { abi, circuit, debug, file_map } = program; + let debug_artifact = DebugArtifact { debug_symbols: vec![debug], file_map }; // Parse the initial witness values from Prover.toml let (inputs_map, _) = diff --git a/tooling/nargo_cli/src/cli/info_cmd.rs b/tooling/nargo_cli/src/cli/info_cmd.rs index e51a0256426..8aa6efc51cd 100644 --- a/tooling/nargo_cli/src/cli/info_cmd.rs +++ b/tooling/nargo_cli/src/cli/info_cmd.rs @@ -174,7 +174,7 @@ fn count_opcodes_and_gates_in_program( np_language: Language, is_opcode_supported: &impl Fn(&Opcode) -> bool, ) -> Result { - let (compiled_program, _) = + let compiled_program = compile_bin_package(package, compile_options, np_language, &is_opcode_supported)?; let (language, _) = backend.get_backend_info()?; diff --git a/tooling/nargo_cli/src/cli/prove_cmd.rs b/tooling/nargo_cli/src/cli/prove_cmd.rs index c451b78add5..03146d3919c 100644 --- a/tooling/nargo_cli/src/cli/prove_cmd.rs +++ b/tooling/nargo_cli/src/cli/prove_cmd.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use acvm::acir::circuit::Opcode; use acvm::Language; use clap::Args; +use nargo::artifacts::debug::DebugArtifact; use nargo::artifacts::program::PreprocessedProgram; use nargo::constants::{PROVER_INPUT_FILE, VERIFIER_INPUT_FILE}; use nargo::package::Package; @@ -101,13 +102,16 @@ pub(crate) fn prove_package( (program, None) } else { - let (program, debug_artifact) = + let program = compile_bin_package(package, compile_options, np_language, &is_opcode_supported)?; let preprocessed_program = PreprocessedProgram { backend: String::from(BACKEND_IDENTIFIER), abi: program.abi, bytecode: program.circuit, }; + let debug_artifact = + DebugArtifact { debug_symbols: vec![program.debug], file_map: program.file_map }; + (preprocessed_program, Some(debug_artifact)) }; diff --git a/tooling/nargo_cli/src/cli/verify_cmd.rs b/tooling/nargo_cli/src/cli/verify_cmd.rs index 9d1a98da4da..452d58ff667 100644 --- a/tooling/nargo_cli/src/cli/verify_cmd.rs +++ b/tooling/nargo_cli/src/cli/verify_cmd.rs @@ -85,7 +85,7 @@ fn verify_package( let preprocessed_program = if circuit_build_path.exists() { read_program_from_file(circuit_build_path)? } else { - let (program, _) = + let program = compile_bin_package(package, compile_options, np_language, &is_opcode_supported)?; PreprocessedProgram { From 93727235d0e20b7bc7d18d1ff7657c4102bd226a Mon Sep 17 00:00:00 2001 From: Tom French Date: Tue, 12 Sep 2023 18:15:02 +0100 Subject: [PATCH 2/7] chore: embed file map into `CompiledContract` --- compiler/noirc_driver/src/contract.rs | 11 ++++- compiler/noirc_driver/src/lib.rs | 5 ++- tooling/nargo_cli/src/cli/compile_cmd.rs | 52 ++++++++++++------------ tooling/nargo_cli/src/cli/info_cmd.rs | 2 +- 4 files changed, 39 insertions(+), 31 deletions(-) diff --git a/compiler/noirc_driver/src/contract.rs b/compiler/noirc_driver/src/contract.rs index a1820ff2e47..69a92764318 100644 --- a/compiler/noirc_driver/src/contract.rs +++ b/compiler/noirc_driver/src/contract.rs @@ -1,8 +1,13 @@ -use crate::program::{deserialize_circuit, serialize_circuit}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + use acvm::acir::circuit::Circuit; +use fm::FileId; use noirc_abi::Abi; use noirc_errors::debug_info::DebugInfo; -use serde::{Deserialize, Serialize}; + +use super::debug::DebugFile; +use crate::program::{deserialize_circuit, serialize_circuit}; /// Describes the types of smart contract functions that are allowed. /// Unlike the similar enum in noirc_frontend, 'open' and 'unconstrained' @@ -28,6 +33,8 @@ pub struct CompiledContract { /// Each of the contract's functions are compiled into a separate `CompiledProgram` /// stored in this `Vector`. pub functions: Vec, + + pub file_map: BTreeMap, } /// Each function in the contract will be compiled diff --git a/compiler/noirc_driver/src/lib.rs b/compiler/noirc_driver/src/lib.rs index 1c8d1b19343..ec569e0e182 100644 --- a/compiler/noirc_driver/src/lib.rs +++ b/compiler/noirc_driver/src/lib.rs @@ -251,7 +251,10 @@ fn compile_contract( } if errors.is_empty() { - Ok(CompiledContract { name: contract.name, functions }) + let debug_infos: Vec<_> = functions.iter().map(|function| function.debug.clone()).collect(); + let file_map = filter_relevant_files(&debug_infos, &context.file_manager); + + Ok(CompiledContract { name: contract.name, functions, file_map }) } else { Err(errors) } diff --git a/tooling/nargo_cli/src/cli/compile_cmd.rs b/tooling/nargo_cli/src/cli/compile_cmd.rs index 447bcdb6f20..6fbf3f80243 100644 --- a/tooling/nargo_cli/src/cli/compile_cmd.rs +++ b/tooling/nargo_cli/src/cli/compile_cmd.rs @@ -79,15 +79,13 @@ pub(crate) fn run( }) .collect(); #[allow(clippy::type_complexity)] - let contract_results: Vec<( - FileManager, - CompilationResult>, - )> = contract_packages - .par_iter() - .map(|package| { - compile_contracts(package, &args.compile_options, np_language, &is_opcode_supported) - }) - .collect(); + let contract_results: Vec<(FileManager, CompilationResult>)> = + contract_packages + .par_iter() + .map(|package| { + compile_contracts(package, &args.compile_options, np_language, &is_opcode_supported) + }) + .collect(); // Report any warnings/errors which were encountered during compilation. let compiled_programs: Vec = program_results @@ -96,7 +94,7 @@ pub(crate) fn run( report_errors(compilation_result, &file_manager, args.compile_options.deny_warnings) }) .collect::>()?; - let compiled_contracts: Vec> = contract_results + let compiled_contracts: Vec> = contract_results .into_iter() .map(|(file_manager, compilation_result)| { report_errors(compilation_result, &file_manager, args.compile_options.deny_warnings) @@ -107,10 +105,8 @@ pub(crate) fn run( for (package, program) in binary_packages.into_iter().zip(compiled_programs) { save_program(program, package, &circuit_dir, args.output_debug); } - for (package, contracts_with_debug_artifacts) in - contract_packages.into_iter().zip(compiled_contracts) - { - save_contracts(contracts_with_debug_artifacts, package, &circuit_dir, args.output_debug); + for (package, compiled_contracts) in contract_packages.into_iter().zip(compiled_contracts) { + save_contracts(compiled_contracts, package, &circuit_dir, args.output_debug); } Ok(()) @@ -139,7 +135,7 @@ pub(crate) fn compile_contract_package( compile_options: &CompileOptions, np_language: Language, is_opcode_supported: &impl Fn(&Opcode) -> bool, -) -> Result, CliError> { +) -> Result, CliError> { let (file_manager, compilation_result) = compile_contracts(package, compile_options, np_language, &is_opcode_supported); let contracts_with_debug_artifacts = @@ -176,7 +172,7 @@ fn compile_contracts( compile_options: &CompileOptions, np_language: Language, is_opcode_supported: &impl Fn(&Opcode) -> bool, -) -> (FileManager, CompilationResult>) { +) -> (FileManager, CompilationResult>) { let (mut context, crate_id) = prepare_package(package); let (contracts, warnings) = match noirc_driver::compile_contracts(&mut context, crate_id, compile_options) { @@ -186,19 +182,12 @@ fn compile_contracts( } }; - let optimized_contracts = try_vecmap(contracts, |contract| { + let optimized_contracts: Vec = try_vecmap(contracts, |contract| { nargo::ops::optimize_contract(contract, np_language, &is_opcode_supported) }) .expect("Backend does not support an opcode that is in the IR"); - let contracts_with_debug_artifacts = vecmap(optimized_contracts, |contract| { - let debug_infos = vecmap(&contract.functions, |func| func.debug.clone()); - let debug_artifact = DebugArtifact::new(debug_infos, &context.file_manager); - - (contract, debug_artifact) - }); - - (context.file_manager, Ok((contracts_with_debug_artifacts, warnings))) + (context.file_manager, Ok((optimized_contracts, warnings))) } fn save_program( @@ -226,7 +215,7 @@ fn save_program( } fn save_contracts( - contracts: Vec<(CompiledContract, DebugArtifact)>, + contracts: Vec, package: &Package, circuit_dir: &Path, output_debug: bool, @@ -236,7 +225,16 @@ fn save_contracts( // are compiled via nargo-core and then the PreprocessedContract is constructed here. // This is due to EACH function needing it's own CRS, PKey, and VKey from the backend. let preprocessed_contracts: Vec<(PreprocessedContract, DebugArtifact)> = - vecmap(contracts, |(contract, debug_artifact)| { + vecmap(contracts, |contract| { + let debug_artifact = DebugArtifact { + debug_symbols: contract + .functions + .iter() + .map(|function| function.debug.clone()) + .collect(), + file_map: contract.file_map, + }; + let preprocessed_functions = vecmap(contract.functions, |func| PreprocessedContractFunction { name: func.name, diff --git a/tooling/nargo_cli/src/cli/info_cmd.rs b/tooling/nargo_cli/src/cli/info_cmd.rs index 8aa6efc51cd..49ae1327c5a 100644 --- a/tooling/nargo_cli/src/cli/info_cmd.rs +++ b/tooling/nargo_cli/src/cli/info_cmd.rs @@ -197,7 +197,7 @@ fn count_opcodes_and_gates_in_contracts( compile_contract_package(package, compile_options, np_language, &is_opcode_supported)?; let (language, _) = backend.get_backend_info()?; - try_vecmap(contracts, |(contract, _)| { + try_vecmap(contracts, |contract| { let functions = try_vecmap(contract.functions, |function| -> Result<_, BackendError> { Ok(FunctionInfo { name: function.name, From e956844409cf92da5370dc087b806f4544a4e9cc Mon Sep 17 00:00:00 2001 From: Tom French Date: Tue, 12 Sep 2023 18:22:13 +0100 Subject: [PATCH 3/7] chore: remove type complexity lint silencing --- tooling/nargo_cli/src/cli/compile_cmd.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tooling/nargo_cli/src/cli/compile_cmd.rs b/tooling/nargo_cli/src/cli/compile_cmd.rs index 6fbf3f80243..d96d96545b7 100644 --- a/tooling/nargo_cli/src/cli/compile_cmd.rs +++ b/tooling/nargo_cli/src/cli/compile_cmd.rs @@ -78,7 +78,6 @@ pub(crate) fn run( compile_program(package, &args.compile_options, np_language, &is_opcode_supported) }) .collect(); - #[allow(clippy::type_complexity)] let contract_results: Vec<(FileManager, CompilationResult>)> = contract_packages .par_iter() From 38b84f6ce92e6630f7029b5c67e6b24918c8f677 Mon Sep 17 00:00:00 2001 From: Tom French Date: Tue, 12 Sep 2023 18:28:35 +0100 Subject: [PATCH 4/7] chore: remove unnecessary explicit type --- tooling/nargo_cli/src/cli/compile_cmd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling/nargo_cli/src/cli/compile_cmd.rs b/tooling/nargo_cli/src/cli/compile_cmd.rs index d96d96545b7..ee7cc21f2e0 100644 --- a/tooling/nargo_cli/src/cli/compile_cmd.rs +++ b/tooling/nargo_cli/src/cli/compile_cmd.rs @@ -181,7 +181,7 @@ fn compile_contracts( } }; - let optimized_contracts: Vec = try_vecmap(contracts, |contract| { + let optimized_contracts = try_vecmap(contracts, |contract| { nargo::ops::optimize_contract(contract, np_language, &is_opcode_supported) }) .expect("Backend does not support an opcode that is in the IR"); From 9f355aeb7638446a0b53db69020a49617b2c11e4 Mon Sep 17 00:00:00 2001 From: Tom French Date: Tue, 12 Sep 2023 16:05:03 +0100 Subject: [PATCH 5/7] feat: compile circuits for `nargo info` in parallel --- tooling/nargo_cli/src/cli/compile_cmd.rs | 61 ++++++++++--------- tooling/nargo_cli/src/cli/info_cmd.rs | 76 ++++++++++-------------- 2 files changed, 60 insertions(+), 77 deletions(-) diff --git a/tooling/nargo_cli/src/cli/compile_cmd.rs b/tooling/nargo_cli/src/cli/compile_cmd.rs index ee7cc21f2e0..9d4db8e6339 100644 --- a/tooling/nargo_cli/src/cli/compile_cmd.rs +++ b/tooling/nargo_cli/src/cli/compile_cmd.rs @@ -63,26 +63,46 @@ pub(crate) fn run( let workspace = resolve_workspace_from_toml(&toml_path, selection)?; let circuit_dir = workspace.target_directory_path(); - let (np_language, is_opcode_supported) = backend.get_backend_info()?; - let (binary_packages, contract_packages): (Vec<_>, Vec<_>) = workspace .members - .iter() + .into_iter() .filter(|package| !package.is_library()) .partition(|package| package.is_binary()); + let (compiled_programs, compiled_contracts) = + compile_workspace(backend, &binary_packages, &contract_packages, &args.compile_options)?; + + // Save build artifacts to disk. + for (package, program) in binary_packages.into_iter().zip(compiled_programs) { + save_program(program, &package, &circuit_dir, args.output_debug); + } + for (package, contracts_with_debug_artifacts) in + contract_packages.into_iter().zip(compiled_contracts) + { + save_contracts(contracts_with_debug_artifacts, &package, &circuit_dir, args.output_debug); + } + + Ok(()) +} + +pub(super) fn compile_workspace( + backend: &Backend, + binary_packages: &[Package], + contract_packages: &[Package], + compile_options: &CompileOptions, +) -> Result<(Vec, Vec>), CliError> { + let (np_language, is_opcode_supported) = backend.get_backend_info()?; + // Compile all of the packages in parallel. let program_results: Vec<(FileManager, CompilationResult)> = binary_packages .par_iter() - .map(|package| { - compile_program(package, &args.compile_options, np_language, &is_opcode_supported) - }) + .map(|package| compile_program(package, compile_options, np_language, &is_opcode_supported)) .collect(); let contract_results: Vec<(FileManager, CompilationResult>)> = contract_packages .par_iter() .map(|package| { - compile_contracts(package, &args.compile_options, np_language, &is_opcode_supported) + compile_contracts(package, compile_options, np_language, &is_opcode_supported) }) .collect(); @@ -90,25 +110,17 @@ pub(crate) fn run( let compiled_programs: Vec = program_results .into_iter() .map(|(file_manager, compilation_result)| { - report_errors(compilation_result, &file_manager, args.compile_options.deny_warnings) + report_errors(compilation_result, &file_manager, compile_options.deny_warnings) }) .collect::>()?; let compiled_contracts: Vec> = contract_results .into_iter() .map(|(file_manager, compilation_result)| { - report_errors(compilation_result, &file_manager, args.compile_options.deny_warnings) + report_errors(compilation_result, &file_manager, compile_options.deny_warnings) }) .collect::>()?; - // Save build artifacts to disk. - for (package, program) in binary_packages.into_iter().zip(compiled_programs) { - save_program(program, package, &circuit_dir, args.output_debug); - } - for (package, compiled_contracts) in contract_packages.into_iter().zip(compiled_contracts) { - save_contracts(compiled_contracts, package, &circuit_dir, args.output_debug); - } - - Ok(()) + Ok((compiled_programs, compiled_contracts)) } pub(crate) fn compile_bin_package( @@ -129,19 +141,6 @@ pub(crate) fn compile_bin_package( Ok(program) } -pub(crate) fn compile_contract_package( - package: &Package, - compile_options: &CompileOptions, - np_language: Language, - is_opcode_supported: &impl Fn(&Opcode) -> bool, -) -> Result, CliError> { - let (file_manager, compilation_result) = - compile_contracts(package, compile_options, np_language, &is_opcode_supported); - let contracts_with_debug_artifacts = - report_errors(compilation_result, &file_manager, compile_options.deny_warnings)?; - Ok(contracts_with_debug_artifacts) -} - fn compile_program( package: &Package, compile_options: &CompileOptions, diff --git a/tooling/nargo_cli/src/cli/info_cmd.rs b/tooling/nargo_cli/src/cli/info_cmd.rs index 49ae1327c5a..dfc1e684832 100644 --- a/tooling/nargo_cli/src/cli/info_cmd.rs +++ b/tooling/nargo_cli/src/cli/info_cmd.rs @@ -1,11 +1,10 @@ -use acvm::acir::circuit::Opcode; use acvm::Language; use acvm_backend_barretenberg::BackendError; use clap::Args; use iter_extended::{try_vecmap, vecmap}; use nargo::package::Package; use nargo_toml::{get_package_manifest, resolve_workspace_from_toml, PackageSelection}; -use noirc_driver::CompileOptions; +use noirc_driver::{CompileOptions, CompiledContract, CompiledProgram}; use noirc_frontend::graph::CrateName; use prettytable::{row, table, Row}; use serde::Serialize; @@ -13,10 +12,7 @@ use serde::Serialize; use crate::backends::Backend; use crate::errors::CliError; -use super::{ - compile_cmd::{compile_bin_package, compile_contract_package}, - NargoConfig, -}; +use super::{compile_cmd::compile_workspace, NargoConfig}; /// Provides detailed information on a circuit /// @@ -52,30 +48,29 @@ pub(crate) fn run( let selection = args.package.map_or(default_selection, PackageSelection::Selected); let workspace = resolve_workspace_from_toml(&toml_path, selection)?; - let mut info_report = InfoReport::default(); - - let (np_language, is_opcode_supported) = backend.get_backend_info()?; - for package in &workspace { - if package.is_contract() { - let contract_info = count_opcodes_and_gates_in_contracts( - backend, - package, - &args.compile_options, - np_language, - &is_opcode_supported, - )?; - info_report.contracts.extend(contract_info); - } else { - let program_info = count_opcodes_and_gates_in_program( - backend, - package, - &args.compile_options, - np_language, - &is_opcode_supported, - )?; - info_report.programs.push(program_info); - } - } + let (binary_packages, contract_packages): (Vec<_>, Vec<_>) = workspace + .members + .into_iter() + .filter(|package| !package.is_library()) + .partition(|package| package.is_binary()); + + let (compiled_programs, compiled_contracts) = + compile_workspace(backend, &binary_packages, &contract_packages, &args.compile_options)?; + + let (np_language, _) = backend.get_backend_info()?; + let program_info = + try_vecmap(binary_packages.iter().zip(compiled_programs), |(package, program)| { + count_opcodes_and_gates_in_program(backend, program, package, np_language) + })?; + + let contract_info = try_vecmap(compiled_contracts, |contracts| { + count_opcodes_and_gates_in_contracts(backend, contracts, np_language) + })?; + + let info_report = InfoReport { + programs: program_info, + contracts: contract_info.into_iter().flatten().collect(), + }; if args.json { // Expose machine-readable JSON data. @@ -169,15 +164,10 @@ impl From for Vec { fn count_opcodes_and_gates_in_program( backend: &Backend, + compiled_program: CompiledProgram, package: &Package, - compile_options: &CompileOptions, - np_language: Language, - is_opcode_supported: &impl Fn(&Opcode) -> bool, + language: Language, ) -> Result { - let compiled_program = - compile_bin_package(package, compile_options, np_language, &is_opcode_supported)?; - let (language, _) = backend.get_backend_info()?; - Ok(ProgramInfo { name: package.name.to_string(), language, @@ -188,16 +178,10 @@ fn count_opcodes_and_gates_in_program( fn count_opcodes_and_gates_in_contracts( backend: &Backend, - package: &Package, - compile_options: &CompileOptions, - np_language: Language, - is_opcode_supported: &impl Fn(&Opcode) -> bool, + compiled_contracts: Vec, + language: Language, ) -> Result, CliError> { - let contracts = - compile_contract_package(package, compile_options, np_language, &is_opcode_supported)?; - let (language, _) = backend.get_backend_info()?; - - try_vecmap(contracts, |contract| { + try_vecmap(compiled_contracts, |contract| { let functions = try_vecmap(contract.functions, |function| -> Result<_, BackendError> { Ok(FunctionInfo { name: function.name, From fd95906f9c4cbe29e5fe6af9886e92f16b0a249d Mon Sep 17 00:00:00 2001 From: Tom French Date: Tue, 12 Sep 2023 23:42:17 +0100 Subject: [PATCH 6/7] chore: parallelise querying circuit size --- tooling/nargo_cli/src/cli/info_cmd.rs | 37 +++++++++++++++++---------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/tooling/nargo_cli/src/cli/info_cmd.rs b/tooling/nargo_cli/src/cli/info_cmd.rs index e8e3b6c5e02..dacc9723fea 100644 --- a/tooling/nargo_cli/src/cli/info_cmd.rs +++ b/tooling/nargo_cli/src/cli/info_cmd.rs @@ -1,12 +1,13 @@ use acvm::Language; use acvm_backend_barretenberg::BackendError; use clap::Args; -use iter_extended::{try_vecmap, vecmap}; +use iter_extended::vecmap; use nargo::package::Package; use nargo_toml::{get_package_manifest, resolve_workspace_from_toml, PackageSelection}; use noirc_driver::{CompileOptions, CompiledContract, CompiledProgram}; use noirc_frontend::graph::CrateName; use prettytable::{row, table, Row}; +use rayon::prelude::*; use serde::Serialize; use crate::backends::Backend; @@ -58,14 +59,18 @@ pub(crate) fn run( compile_workspace(backend, &binary_packages, &contract_packages, &args.compile_options)?; let (np_language, _) = backend.get_backend_info()?; - let program_info = - try_vecmap(binary_packages.iter().zip(compiled_programs), |(package, program)| { - count_opcodes_and_gates_in_program(backend, program, package, np_language) - })?; + let program_info = binary_packages + .into_par_iter() + .zip(compiled_programs) + .map(|(package, program)| { + count_opcodes_and_gates_in_program(backend, program, &package, np_language) + }) + .collect::>()?; - let contract_info = try_vecmap(compiled_contracts, |contract| { - count_opcodes_and_gates_in_contract(backend, contract, np_language) - })?; + let contract_info = compiled_contracts + .into_par_iter() + .map(|contract| count_opcodes_and_gates_in_contract(backend, contract, np_language)) + .collect::>()?; let info_report = InfoReport { programs: program_info, contracts: contract_info }; @@ -178,13 +183,17 @@ fn count_opcodes_and_gates_in_contract( contract: CompiledContract, language: Language, ) -> Result { - let functions = try_vecmap(contract.functions, |function| -> Result<_, BackendError> { - Ok(FunctionInfo { - name: function.name, - acir_opcodes: function.bytecode.opcodes.len(), - circuit_size: backend.get_exact_circuit_size(&function.bytecode)?, + let functions = contract + .functions + .into_par_iter() + .map(|function| -> Result<_, BackendError> { + Ok(FunctionInfo { + name: function.name, + acir_opcodes: function.bytecode.opcodes.len(), + circuit_size: backend.get_exact_circuit_size(&function.bytecode)?, + }) }) - })?; + .collect::>()?; Ok(ContractInfo { name: contract.name, language, functions }) } From 0dae6125ebedafff10231394d51d5e16a7e028c4 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 13 Sep 2023 00:18:08 +0100 Subject: [PATCH 7/7] fix: respect package selection --- tooling/nargo_cli/src/cli/compile_cmd.rs | 2 +- tooling/nargo_cli/src/cli/info_cmd.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tooling/nargo_cli/src/cli/compile_cmd.rs b/tooling/nargo_cli/src/cli/compile_cmd.rs index 2069bda308f..c769cb68ba5 100644 --- a/tooling/nargo_cli/src/cli/compile_cmd.rs +++ b/tooling/nargo_cli/src/cli/compile_cmd.rs @@ -64,9 +64,9 @@ pub(crate) fn run( let circuit_dir = workspace.target_directory_path(); let (binary_packages, contract_packages): (Vec<_>, Vec<_>) = workspace - .members .into_iter() .filter(|package| !package.is_library()) + .cloned() .partition(|package| package.is_binary()); let (compiled_programs, compiled_contracts) = diff --git a/tooling/nargo_cli/src/cli/info_cmd.rs b/tooling/nargo_cli/src/cli/info_cmd.rs index dacc9723fea..ffa522d25b4 100644 --- a/tooling/nargo_cli/src/cli/info_cmd.rs +++ b/tooling/nargo_cli/src/cli/info_cmd.rs @@ -50,9 +50,9 @@ pub(crate) fn run( let workspace = resolve_workspace_from_toml(&toml_path, selection)?; let (binary_packages, contract_packages): (Vec<_>, Vec<_>) = workspace - .members .into_iter() .filter(|package| !package.is_library()) + .cloned() .partition(|package| package.is_binary()); let (compiled_programs, compiled_contracts) =