diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a6268b2c75f..9253b77a6be4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -521,6 +521,18 @@ jobs: command: cond_spot_run_build yarn-project-test 64 aztec_manifest_key: yarn-project-test + prover-client-test: + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small + steps: + - *checkout + - *setup_env + - run: + name: "Build and test" + command: cond_spot_run_build prover-client-test 128 + aztec_manifest_key: prover-client-test + aztec-package: machine: image: default @@ -1429,6 +1441,7 @@ workflows: - end-to-end: *defaults_yarn_project - aztec-faucet: *defaults_yarn_project_pre_join - build-docs: *defaults_yarn_project_pre_join + - prover-client-test: *defaults_yarn_project - yarn-project-test: *defaults_yarn_project - yarn-project-x86_64: *defaults_yarn_project_pre_join - yarn-project-arm64: *defaults_yarn_project_pre_join @@ -1581,6 +1594,7 @@ workflows: - yellow-paper - noir-packages-tests - yarn-project-test + - prover-client-test <<: *defaults # Benchmark jobs. diff --git a/build_manifest.yml b/build_manifest.yml index e650233f6d1e..afba6e3820f7 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -160,6 +160,23 @@ yarn-project-test: - l1-contracts - noir-projects - barretenberg-x86_64-linux-clang + - noir + +# Runs all prover-client checks and tests. +prover-client-test: + buildDir: yarn-project + projectDir: yarn-project/prover-client + dockerfile: Dockerfile.test + rebuildPatterns: + - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml|snap)$ + - ^yarn-project/Dockerfile$ + dependencies: + - bb.js + - noir-packages + - l1-contracts + - noir-projects + - barretenberg-x86_64-linux-clang + - noir # Builds all of yarn-project, with all developer dependencies. # Creates a runnable container used to run tests and formatting checks. diff --git a/cspell.json b/cspell.json index 9180d507c568..01c51565daec 100644 --- a/cspell.json +++ b/cspell.json @@ -18,6 +18,7 @@ "bbfree", "bbmalloc", "benesjan", + "Bincode", "bleurgh", "bodyparser", "bootnode", diff --git a/noir/noir-repo/tooling/acvm_cli/src/cli/execute_cmd.rs b/noir/noir-repo/tooling/acvm_cli/src/cli/execute_cmd.rs index 86e7277451fb..4e36dbd1f225 100644 --- a/noir/noir-repo/tooling/acvm_cli/src/cli/execute_cmd.rs +++ b/noir/noir-repo/tooling/acvm_cli/src/cli/execute_cmd.rs @@ -6,11 +6,10 @@ use bn254_blackbox_solver::Bn254BlackBoxSolver; use clap::Args; use crate::cli::fs::inputs::{read_bytecode_from_file, read_inputs_from_file}; -use crate::cli::fs::witness::save_witness_to_dir; use crate::errors::CliError; use nargo::ops::{execute_program, DefaultForeignCallExecutor}; -use super::fs::witness::create_output_witness_string; +use super::fs::witness::{create_output_witness_string, save_witness_to_dir}; /// Executes a circuit to calculate its return value #[derive(Debug, Clone, Args)] @@ -46,9 +45,9 @@ fn run_command(args: ExecuteCommand) -> Result { )?; if args.output_witness.is_some() { save_witness_to_dir( - &output_witness_string, - &args.working_directory, + output_witness, &args.output_witness.unwrap(), + &args.working_directory, )?; } Ok(output_witness_string) diff --git a/noir/noir-repo/tooling/acvm_cli/src/cli/fs/witness.rs b/noir/noir-repo/tooling/acvm_cli/src/cli/fs/witness.rs index 2daaa5a3a584..30ef4278f4bb 100644 --- a/noir/noir-repo/tooling/acvm_cli/src/cli/fs/witness.rs +++ b/noir/noir-repo/tooling/acvm_cli/src/cli/fs/witness.rs @@ -5,24 +5,29 @@ use std::{ path::{Path, PathBuf}, }; -use acvm::acir::native_types::WitnessMap; +use acvm::acir::native_types::{WitnessMap, WitnessStack}; use crate::errors::{CliError, FilesystemError}; -/// Saves the provided output witnesses to a toml file created at the given location -pub(crate) fn save_witness_to_dir>( - output_witness: &String, - witness_dir: P, - file_name: &String, -) -> Result { - let witness_path = witness_dir.as_ref().join(file_name); +fn create_named_dir(named_dir: &Path, name: &str) -> PathBuf { + std::fs::create_dir_all(named_dir) + .unwrap_or_else(|_| panic!("could not create the `{name}` directory")); + + PathBuf::from(named_dir) +} - let mut file = File::create(&witness_path) - .map_err(|_| FilesystemError::OutputWitnessCreationFailed(file_name.clone()))?; - write!(file, "{}", output_witness) - .map_err(|_| FilesystemError::OutputWitnessWriteFailed(file_name.clone()))?; +fn write_to_file(bytes: &[u8], path: &Path) -> String { + let display = path.display(); - Ok(witness_path) + let mut file = match File::create(path) { + Err(why) => panic!("couldn't create {display}: {why}"), + Ok(file) => file, + }; + + match file.write_all(bytes) { + Err(why) => panic!("couldn't write to {display}: {why}"), + Ok(_) => display.to_string(), + } } /// Creates a toml representation of the provided witness map @@ -34,3 +39,19 @@ pub(crate) fn create_output_witness_string(witnesses: &WitnessMap) -> Result>( + witnesses: WitnessStack, + witness_name: &str, + witness_dir: P, +) -> Result { + create_named_dir(witness_dir.as_ref(), "witness"); + let witness_path = witness_dir.as_ref().join(witness_name).with_extension("gz"); + + let buf: Vec = witnesses + .try_into() + .map_err(|_op| FilesystemError::OutputWitnessCreationFailed(witness_name.to_string()))?; + write_to_file(buf.as_slice(), &witness_path); + + Ok(witness_path) +} diff --git a/noir/noir-repo/tooling/acvm_cli/src/errors.rs b/noir/noir-repo/tooling/acvm_cli/src/errors.rs index 923046410eac..8bc79347159a 100644 --- a/noir/noir-repo/tooling/acvm_cli/src/errors.rs +++ b/noir/noir-repo/tooling/acvm_cli/src/errors.rs @@ -20,9 +20,6 @@ pub(crate) enum FilesystemError { #[error(" Error: failed to create output witness file {0}.")] OutputWitnessCreationFailed(String), - - #[error(" Error: failed to write output witness file {0}.")] - OutputWitnessWriteFailed(String), } #[derive(Debug, Error)] diff --git a/noir/noir-repo/tooling/noirc_abi_wasm/src/lib.rs b/noir/noir-repo/tooling/noirc_abi_wasm/src/lib.rs index ce15f6d502ee..fad5abaebba5 100644 --- a/noir/noir-repo/tooling/noirc_abi_wasm/src/lib.rs +++ b/noir/noir-repo/tooling/noirc_abi_wasm/src/lib.rs @@ -5,7 +5,7 @@ // See Cargo.toml for explanation. use getrandom as _; -use acvm::acir::native_types::WitnessMap; +use acvm::acir::native_types::{WitnessMap, WitnessStack}; use iter_extended::try_btree_map; use noirc_abi::{ errors::InputParserError, @@ -113,3 +113,12 @@ pub fn abi_decode(abi: JsAbi, witness_map: JsWitnessMap) -> Result::from_serde(&return_struct) .map_err(|err| err.to_string().into()) } + +#[wasm_bindgen(js_name = serializeWitness)] +pub fn serialise_witness(witness_map: JsWitnessMap) -> Result, JsAbiError> { + console_error_panic_hook::set_once(); + let converted_witness: WitnessMap = witness_map.into(); + let witness_stack: WitnessStack = converted_witness.into(); + let output = witness_stack.try_into(); + output.map_err(|_| JsAbiError::new("Failed to convert to Vec".to_string())) +} diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index bf2b43f1f0f5..7223c9b057ae 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -13,7 +13,7 @@ COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects -# We want the native ACVM binary +# We want the native ACVM and BB binaries COPY --from=noir /usr/src/noir/noir-repo/target/release/acvm /usr/src/noir/noir-repo/target/release/acvm COPY --from=barretenberg /usr/src/barretenberg/cpp/build/bin/bb /usr/src/barretenberg/cpp/build/bin/bb diff --git a/yarn-project/Dockerfile.test b/yarn-project/Dockerfile.test index c5b9b4ab89b6..d5999b264caf 100644 --- a/yarn-project/Dockerfile.test +++ b/yarn-project/Dockerfile.test @@ -3,6 +3,7 @@ FROM --platform=linux/amd64 aztecprotocol/noir-packages as noir-packages FROM --platform=linux/amd64 aztecprotocol/l1-contracts as contracts FROM --platform=linux/amd64 aztecprotocol/noir-projects as noir-projects FROM --platform=linux/amd64 aztecprotocol/barretenberg-x86_64-linux-clang as barretenberg +FROM aztecprotocol/noir as noir FROM node:18.19.0 as builder RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean @@ -12,6 +13,8 @@ COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects +# We want the native ACVM and BB binaries +COPY --from=noir /usr/src/noir/noir-repo/target/release/acvm /usr/src/noir/noir-repo/target/release/acvm COPY --from=barretenberg /usr/src/barretenberg/cpp/build/bin/bb /usr/src/barretenberg/cpp/build/bin/bb WORKDIR /usr/src/yarn-project diff --git a/yarn-project/circuit-types/src/tx/processed_tx.ts b/yarn-project/circuit-types/src/tx/processed_tx.ts index a70335cb85f0..4c041c1d5f52 100644 --- a/yarn-project/circuit-types/src/tx/processed_tx.ts +++ b/yarn-project/circuit-types/src/tx/processed_tx.ts @@ -12,10 +12,34 @@ import { type Header, KernelCircuitPublicInputs, type Proof, + type PublicKernelCircuitPrivateInputs, type PublicKernelCircuitPublicInputs, + type PublicKernelTailCircuitPrivateInputs, makeEmptyProof, } from '@aztec/circuits.js'; +/** + * Used to communicate to the prover which type of circuit to prove + */ +export enum PublicKernelType { + SETUP, + APP_LOGIC, + TEARDOWN, + TAIL, +} + +export type PublicKernelTailRequest = { + type: PublicKernelType.TAIL; + inputs: PublicKernelTailCircuitPrivateInputs; +}; + +export type PublicKernelNonTailRequest = { + type: PublicKernelType.SETUP | PublicKernelType.APP_LOGIC | PublicKernelType.TEARDOWN; + inputs: PublicKernelCircuitPrivateInputs; +}; + +export type PublicKernelRequest = PublicKernelTailRequest | PublicKernelNonTailRequest; + /** * Represents a tx that has been processed by the sequencer public processor, * so its kernel circuit public inputs are filled in. @@ -38,6 +62,11 @@ export type ProcessedTx = Pick { seed + 0x500, ); - const processedTx = makeProcessedTx(tx, kernelOutput, makeProof()); + const processedTx = makeProcessedTx(tx, kernelOutput, makeProof(), []); processedTx.data.end.newNoteHashes = makeTuple(MAX_NEW_NOTE_HASHES_PER_TX, fr, seed + 0x100); processedTx.data.end.newNullifiers = makeTuple(MAX_NEW_NULLIFIERS_PER_TX, fr, seed + 0x200); diff --git a/yarn-project/noir-protocol-circuits-types/src/index.ts b/yarn-project/noir-protocol-circuits-types/src/index.ts index dab4986b71d7..568a5071356f 100644 --- a/yarn-project/noir-protocol-circuits-types/src/index.ts +++ b/yarn-project/noir-protocol-circuits-types/src/index.ts @@ -29,7 +29,7 @@ import { createBlackBoxSolver, executeCircuitWithBlackBoxSolver, } from '@noir-lang/acvm_js'; -import { type Abi, abiDecode, abiEncode } from '@noir-lang/noirc_abi'; +import { type Abi, abiDecode, abiEncode, serializeWitness } from '@noir-lang/noirc_abi'; import { type WitnessMap } from '@noir-lang/types'; import BaseParityJson from './target/parity_base.json' assert { type: 'json' }; @@ -46,6 +46,7 @@ import PublicKernelAppLogicSimulatedJson from './target/public_kernel_app_logic_ import PublicKernelSetupSimulatedJson from './target/public_kernel_setup_simulated.json' assert { type: 'json' }; import PublicKernelTailSimulatedJson from './target/public_kernel_tail_simulated.json' assert { type: 'json' }; import PublicKernelTeardownSimulatedJson from './target/public_kernel_teardown_simulated.json' assert { type: 'json' }; +import BaseRollupJson from './target/rollup_base.json' assert { type: 'json' }; import BaseRollupSimulatedJson from './target/rollup_base_simulated.json' assert { type: 'json' }; import MergeRollupJson from './target/rollup_merge.json' assert { type: 'json' }; import RootRollupJson from './target/rollup_root.json' assert { type: 'json' }; @@ -126,12 +127,65 @@ export const BaseParityArtifact = BaseParityJson as NoirCompiledCircuit; export const RootParityArtifact = RootParityJson as NoirCompiledCircuit; -export const BaseRollupArtifact = BaseRollupSimulatedJson as NoirCompiledCircuit; +export const SimulatedBaseRollupArtifact = BaseRollupSimulatedJson as NoirCompiledCircuit; + +export const BaseRollupArtifact = BaseRollupJson as NoirCompiledCircuit; export const MergeRollupArtifact = MergeRollupJson as NoirCompiledCircuit; export const RootRollupArtifact = RootRollupJson as NoirCompiledCircuit; +export type ServerProtocolArtifact = + | 'PublicKernelSetupArtifact' + | 'PublicKernelAppLogicArtifact' + | 'PublicKernelTeardownArtifact' + | 'PublicKernelTailArtifact' + | 'BaseParityArtifact' + | 'RootParityArtifact' + | 'BaseRollupArtifact' + | 'MergeRollupArtifact' + | 'RootRollupArtifact'; + +export type ClientProtocolArtifact = + | 'PrivateKernelInitArtifact' + | 'PrivateKernelInnerArtifact' + | 'PrivateKernelTailArtifact'; + +export type ProtocolArtifact = ServerProtocolArtifact | ClientProtocolArtifact; + +export const ServerCircuitArtifacts: Record = { + PublicKernelSetupArtifact: PublicKernelSetupArtifact, + PublicKernelAppLogicArtifact: PublicKernelAppLogicArtifact, + PublicKernelTeardownArtifact: PublicKernelTeardownArtifact, + PublicKernelTailArtifact: PublicKernelTailArtifact, + BaseParityArtifact: BaseParityArtifact, + RootParityArtifact: RootParityArtifact, + BaseRollupArtifact: BaseRollupArtifact, + MergeRollupArtifact: MergeRollupArtifact, + RootRollupArtifact: RootRollupArtifact, +}; + +export const ClientCircuitArtifacts: Record = { + PrivateKernelInitArtifact: PrivateKernelInitArtifact, + PrivateKernelInnerArtifact: PrivateKernelInnerArtifact, + PrivateKernelTailArtifact: PrivateKernelTailArtifact, +}; + +export const ProtocolCircuitArtifacts: Record = { + PrivateKernelInitArtifact: PrivateKernelInitArtifact, + PrivateKernelInnerArtifact: PrivateKernelInnerArtifact, + PrivateKernelTailArtifact: PrivateKernelTailArtifact, + PublicKernelSetupArtifact: PublicKernelSetupArtifact, + PublicKernelAppLogicArtifact: PublicKernelAppLogicArtifact, + PublicKernelTeardownArtifact: PublicKernelTeardownArtifact, + PublicKernelTailArtifact: PublicKernelTailArtifact, + BaseParityArtifact: BaseParityArtifact, + RootParityArtifact: RootParityArtifact, + BaseRollupArtifact: BaseRollupArtifact, + MergeRollupArtifact: MergeRollupArtifact, + RootRollupArtifact: RootRollupArtifact, +}; + let solver: Promise; const getSolver = (): Promise => { @@ -141,6 +195,10 @@ const getSolver = (): Promise => { return solver; }; +export function serializeInputWitness(witness: WitnessMap) { + return serializeWitness(witness); +} + /** * Executes the init private kernel. * @param privateKernelInitCircuitPrivateInputs - The private inputs to the initial private kernel. @@ -236,6 +294,17 @@ export function convertRootParityInputsToWitnessMap(inputs: RootParityInputs): W * @returns The witness map */ export function convertBaseRollupInputsToWitnessMap(inputs: BaseRollupInputs): WitnessMap { + const mapped = mapBaseRollupInputsToNoir(inputs); + const initialWitnessMap = abiEncode(BaseRollupJson.abi as Abi, { inputs: mapped as any }); + return initialWitnessMap; +} + +/** + * Converts the inputs of the simulated base rollup circuit into a witness map. + * @param inputs - The base rollup inputs. + * @returns The witness map + */ +export function convertSimulatedBaseRollupInputsToWitnessMap(inputs: BaseRollupInputs): WitnessMap { const mapped = mapBaseRollupInputsToNoir(inputs); const initialWitnessMap = abiEncode(BaseRollupSimulatedJson.abi as Abi, { inputs: mapped as any }); return initialWitnessMap; @@ -306,6 +375,21 @@ export function convertPublicTailInputsToWitnessMap(inputs: PublicKernelTailCirc return initialWitnessMap; } +/** + * Converts the outputs of the simulated base rollup circuit from a witness map. + * @param outputs - The base rollup outputs as a witness map. + * @returns The public inputs. + */ +export function convertSimulatedBaseRollupOutputsFromWitnessMap(outputs: WitnessMap): BaseOrMergeRollupPublicInputs { + // Decode the witness map into two fields, the return values and the inputs + const decodedInputs: DecodedInputs = abiDecode(BaseRollupSimulatedJson.abi as Abi, outputs); + + // Cast the inputs as the return type + const returnType = decodedInputs.return_value as BaseRollupReturnType; + + return mapBaseOrMergeRollupPublicInputsFromNoir(returnType); +} + /** * Converts the outputs of the base rollup circuit from a witness map. * @param outputs - The base rollup outputs as a witness map. @@ -313,7 +397,7 @@ export function convertPublicTailInputsToWitnessMap(inputs: PublicKernelTailCirc */ export function convertBaseRollupOutputsFromWitnessMap(outputs: WitnessMap): BaseOrMergeRollupPublicInputs { // Decode the witness map into two fields, the return values and the inputs - const decodedInputs: DecodedInputs = abiDecode(BaseRollupSimulatedJson.abi as Abi, outputs); + const decodedInputs: DecodedInputs = abiDecode(BaseRollupJson.abi as Abi, outputs); // Cast the inputs as the return type const returnType = decodedInputs.return_value as BaseRollupReturnType; diff --git a/yarn-project/package.json b/yarn-project/package.json index a74c1f2f6286..70f765df28f3 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -10,7 +10,7 @@ "formatting:fix": "FORCE_COLOR=true yarn workspaces foreach -p -v run formatting:fix", "lint": "yarn eslint --cache --ignore-pattern l1-artifacts .", "format": "yarn prettier --cache -w .", - "test": "FORCE_COLOR=true yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end -p -v run test", + "test": "FORCE_COLOR=true yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end --exclude @aztec/prover-client -p -v run test", "build": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose --exclude @aztec/aztec3-packages --exclude @aztec/docs run build", "build:fast": "yarn generate && tsc -b", "build:dev": "./watch.sh", diff --git a/yarn-project/prover-client/Dockerfile.test b/yarn-project/prover-client/Dockerfile.test new file mode 100644 index 000000000000..45895a749e85 --- /dev/null +++ b/yarn-project/prover-client/Dockerfile.test @@ -0,0 +1,39 @@ +FROM --platform=linux/amd64 aztecprotocol/bb.js as bb.js +FROM --platform=linux/amd64 aztecprotocol/noir-packages as noir-packages +FROM --platform=linux/amd64 aztecprotocol/l1-contracts as contracts +FROM --platform=linux/amd64 aztecprotocol/noir-projects as noir-projects +FROM --platform=linux/amd64 aztecprotocol/barretenberg-x86_64-linux-clang as barretenberg +FROM aztecprotocol/noir as noir + +FROM node:18.19.0 as builder +RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean + +# Copy in portalled packages. +COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts +COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages +COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts +COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects +# We want the native ACVM and BB binaries +COPY --from=noir /usr/src/noir/noir-repo/target/release/acvm /usr/src/noir/noir-repo/target/release/acvm +COPY --from=barretenberg /usr/src/barretenberg/cpp/build/bin/bb /usr/src/barretenberg/cpp/build/bin/bb + +WORKDIR /usr/src/yarn-project +COPY . . + +# We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they +# walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution +# correctly for portalled packages, is to use --preserve-symlinks when running node. +# This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic +# by initialising the module more than once. So at present I don't see a viable alternative. +RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules + +# TODO: Replace puppeteer with puppeteer-core to avoid this. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true + +RUN ./bootstrap.sh +RUN cd prover-client && LOG_LEVEL=verbose ACVM_WORKING_DIRECTORY='/tmp/acvm' BB_WORKING_DIRECTORY='/tmp/bb' yarn test + +# Avoid pushing some huge container back to ecr. +FROM scratch +COPY --from=builder /usr/src/yarn-project/README.md /usr/src/yarn-project/README.md diff --git a/yarn-project/prover-client/package.json b/yarn-project/prover-client/package.json index 6071ba9ad4ae..a831caf6783f 100644 --- a/yarn-project/prover-client/package.json +++ b/yarn-project/prover-client/package.json @@ -3,6 +3,9 @@ "version": "0.1.0", "type": "module", "exports": "./dest/index.js", + "bin": { + "bb-cli": "./dest/bb/index.js" + }, "typedocOptions": { "entryPoints": [ "./src/index.ts" @@ -16,6 +19,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", + "bb": "node --no-warnings ./dest/bb/index.js", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests" }, "inherits": [ @@ -44,7 +48,9 @@ "@aztec/noir-protocol-circuits-types": "workspace:^", "@aztec/simulator": "workspace:^", "@aztec/world-state": "workspace:^", + "commander": "^9.0.0", "lodash.chunk": "^4.2.0", + "source-map-support": "^0.5.21", "tslib": "^2.4.0" }, "devDependencies": { @@ -52,6 +58,7 @@ "@types/jest": "^29.5.0", "@types/memdown": "^3.0.0", "@types/node": "^18.7.23", + "@types/source-map-support": "^0.5.10", "jest": "^29.5.0", "jest-mock-extended": "^3.0.3", "ts-node": "^10.9.1", diff --git a/yarn-project/prover-client/src/bb/cli.ts b/yarn-project/prover-client/src/bb/cli.ts new file mode 100644 index 000000000000..5358355d6ae6 --- /dev/null +++ b/yarn-project/prover-client/src/bb/cli.ts @@ -0,0 +1,92 @@ +import { type LogFn } from '@aztec/foundation/log'; +import { type ProtocolArtifact, ProtocolCircuitArtifacts } from '@aztec/noir-protocol-circuits-types'; + +import { Command } from 'commander'; +import * as fs from 'fs/promises'; + +import { generateKeyForNoirCircuit } from './execute.js'; + +const { BB_WORKING_DIRECTORY, BB_BINARY_PATH } = process.env; + +/** + * Returns commander program that defines the CLI. + * @param log - Console logger. + * @returns The CLI. + */ +export function getProgram(log: LogFn): Command { + const program = new Command(); + + program.name('bb-cli').description('CLI for interacting with Barretenberg.'); + + program + .command('protocol-circuits') + .description('Lists the available protocol circuit artifacts') + .action(() => { + log(Object.keys(ProtocolCircuitArtifacts).reduce((prev: string, x: string) => prev.concat(`\n${x}`))); + }); + + program + .command('write_pk') + .description('Generates the proving key for the specified circuit') + .requiredOption( + '-w, --working-directory ', + 'A directory to use for storing input/output files', + BB_WORKING_DIRECTORY, + ) + .requiredOption('-b, --bb-path ', 'The path to the BB binary', BB_BINARY_PATH) + .requiredOption('-c, --circuit ', 'The name of a protocol circuit') + .action(async options => { + const compiledCircuit = ProtocolCircuitArtifacts[options.circuit as ProtocolArtifact]; + if (!compiledCircuit) { + log(`Failed to find circuit ${options.circuit}`); + return; + } + try { + await fs.access(options.workingDirectory); + } catch (error) { + log(`Working directory does not exist`); + return; + } + await generateKeyForNoirCircuit( + options.bbPath, + options.workingDirectory, + options.circuit, + compiledCircuit, + 'pk', + log, + ); + }); + + program + .command('write_vk') + .description('Generates the verification key for the specified circuit') + .requiredOption( + '-w, --working-directory ', + 'A directory to use for storing input/output files', + BB_WORKING_DIRECTORY, + ) + .requiredOption('-b, --bb-path ', 'The path to the BB binary', BB_BINARY_PATH) + .requiredOption('-c, --circuit ', 'The name of a protocol circuit') + .action(async options => { + const compiledCircuit = ProtocolCircuitArtifacts[options.circuit as ProtocolArtifact]; + if (!compiledCircuit) { + log(`Failed to find circuit ${options.circuit}`); + return; + } + try { + await fs.access(options.workingDirectory); + } catch (error) { + log(`Working directory does not exist`); + return; + } + await generateKeyForNoirCircuit( + options.bbPath, + options.workingDirectory, + options.circuit, + compiledCircuit, + 'vk', + log, + ); + }); + return program; +} diff --git a/yarn-project/prover-client/src/bb/execute.ts b/yarn-project/prover-client/src/bb/execute.ts new file mode 100644 index 000000000000..87cfba93af7a --- /dev/null +++ b/yarn-project/prover-client/src/bb/execute.ts @@ -0,0 +1,258 @@ +import { sha256 } from '@aztec/foundation/crypto'; +import { type LogFn } from '@aztec/foundation/log'; +import { Timer } from '@aztec/foundation/timer'; +import { type NoirCompiledCircuit } from '@aztec/types/noir'; + +import * as proc from 'child_process'; +import * as fs from 'fs/promises'; + +export enum BB_RESULT { + SUCCESS, + FAILURE, + ALREADY_PRESENT, +} + +export type BBSuccess = { + status: BB_RESULT.SUCCESS | BB_RESULT.ALREADY_PRESENT; + duration: number; + path?: string; +}; + +export type BBFailure = { + status: BB_RESULT.FAILURE; + reason: string; +}; + +export type BBResult = BBSuccess | BBFailure; + +/** + * Invokes the Barretenberg binary with the provided command and args + * @param pathToBB - The path to the BB binary + * @param command - The command to execute + * @param args - The arguments to pass + * @param logger - A log function + * @param resultParser - An optional handler for detecting success or failure + * @returns The completed partial witness outputted from the circuit + */ +export function executeBB( + pathToBB: string, + command: string, + args: string[], + logger: LogFn, + resultParser = (code: number) => code === 0, +) { + return new Promise((resolve, reject) => { + // spawn the bb process + const acvm = proc.spawn(pathToBB, [command, ...args]); + acvm.stdout.on('data', data => { + const message = data.toString('utf-8').replace(/\n$/, ''); + logger(message); + }); + acvm.stderr.on('data', data => { + const message = data.toString('utf-8').replace(/\n$/, ''); + logger(message); + }); + acvm.on('close', (code: number) => { + if (resultParser(code)) { + resolve(BB_RESULT.SUCCESS); + } else { + reject(); + } + }); + }).catch(_ => BB_RESULT.FAILURE); +} + +const bytecodeHashFilename = 'bytecode_hash'; +const bytecodeFilename = 'bytecode'; +const proofFileName = 'proof'; + +/** + * Used for generating either a proving or verification key, will exit early if the key already exists + * It assumes the provided working directory is one where the caller wishes to maintain a permanent set of keys + * It is not considered a temporary directory + * @param pathToBB - The full path to the bb binary + * @param workingDirectory - The directory into which the key should be created + * @param circuitName - An identifier for the circuit + * @param compiledCircuit - The compiled circuit + * @param key - The type of key, either 'pk' or 'vk' + * @param log - A logging function + * @param force - Force the key to be regenerated even if it already exists + * @returns An instance of BBResult + */ +export async function generateKeyForNoirCircuit( + pathToBB: string, + workingDirectory: string, + circuitName: string, + compiledCircuit: NoirCompiledCircuit, + key: 'vk' | 'pk', + log: LogFn, + force = false, +): Promise { + const bytecode = Buffer.from(compiledCircuit.bytecode, 'base64'); + + // The key generation is written to e.g. /workingDirectory/pk/BaseParityArtifact/pk + // The bytecode hash file is also written here as /workingDirectory/pk/BaseParityArtifact/bytecode-hash + // The bytecode is written to e.g. /workingDirectory/pk/BaseParityArtifact/bytecode + // The bytecode is removed after the key is generated, leaving just the hash file + const circuitOutputDirectory = `${workingDirectory}/${key}/${circuitName}`; + const bytecodeHashPath = `${circuitOutputDirectory}/${bytecodeHashFilename}`; + const bytecodePath = `${circuitOutputDirectory}/${bytecodeFilename}`; + const bytecodeHash = sha256(bytecode); + + const outputPath = `${circuitOutputDirectory}/${key}`; + + // ensure the directory exists + await fs.mkdir(circuitOutputDirectory, { recursive: true }); + + // Generate the key if we have been told to, or there is no bytecode hash + let mustRegenerate = + force || + (await fs + .access(bytecodeHashPath, fs.constants.R_OK) + .then(_ => false) + .catch(_ => true)); + + if (!mustRegenerate) { + // Check to see if the bytecode hash has changed from the stored value + const data: Buffer = await fs.readFile(bytecodeHashPath).catch(_ => Buffer.alloc(0)); + mustRegenerate = data.length == 0 || !data.equals(bytecodeHash); + } + + if (!mustRegenerate) { + // No need to generate, early out + return { status: BB_RESULT.ALREADY_PRESENT, duration: 0, path: outputPath }; + } + + // Check we have access to bb + const binaryPresent = await fs + .access(pathToBB, fs.constants.R_OK) + .then(_ => true) + .catch(_ => false); + if (!binaryPresent) { + return { status: BB_RESULT.FAILURE, reason: `Failed to find bb binary at ${pathToBB}` }; + } + + // We are now going to generate the key + try { + // Write the bytecode to the working directory + await fs.writeFile(bytecodePath, bytecode); + + // args are the output path and the input bytecode path + const args = ['-o', outputPath, '-b', bytecodePath]; + const timer = new Timer(); + const result = await executeBB(pathToBB, `write_${key}`, args, log); + const duration = timer.ms(); + // Cleanup the bytecode file + await fs.rm(bytecodePath, { force: true }); + if (result == BB_RESULT.SUCCESS) { + // Store the bytecode hash so we don't need to regenerate at a later time + await fs.writeFile(bytecodeHashPath, bytecodeHash); + return { status: BB_RESULT.SUCCESS, duration, path: outputPath }; + } + // Not a great error message here but it is difficult to decipher what comes from bb + return { status: BB_RESULT.FAILURE, reason: `Failed to generate key` }; + } catch (error) { + return { status: BB_RESULT.FAILURE, reason: `${error}` }; + } +} + +/** + * Used for generating proofs of noir circuits. + * It is assumed that the working directory is a temporary and/or random directory used solely for generating this proof. + * @param pathToBB - The full path to the bb binary + * @param workingDirectory - A working directory for use by bb + * @param circuitName - An identifier for the circuit + * @param compiledCircuit - The compiled circuit + * @param inputWitnessFile - The circuit input witness + * @param log - A logging function + * @returns An object containing a result indication, the location of the proof and the duration taken + */ +export async function generateProof( + pathToBB: string, + workingDirectory: string, + circuitName: string, + compiledCircuit: NoirCompiledCircuit, + inputWitnessFile: string, + log: LogFn, +): Promise { + // Check that the working directory exists + try { + await fs.access(workingDirectory); + } catch (error) { + return { status: BB_RESULT.FAILURE, reason: `Working directory ${workingDirectory} does not exist` }; + } + + // The bytecode is written to e.g. /workingDirectory/BaseParityArtifact-bytecode + const bytecodePath = `${workingDirectory}/${circuitName}-bytecode`; + const bytecode = Buffer.from(compiledCircuit.bytecode, 'base64'); + + // The proof is written to e.g. /workingDirectory/proof + const outputPath = `${workingDirectory}/${proofFileName}`; + + const binaryPresent = await fs + .access(pathToBB, fs.constants.R_OK) + .then(_ => true) + .catch(_ => false); + if (!binaryPresent) { + return { status: BB_RESULT.FAILURE, reason: `Failed to find bb binary at ${pathToBB}` }; + } + + try { + // Write the bytecode to the working directory + await fs.writeFile(bytecodePath, bytecode); + const args = ['-o', outputPath, '-b', bytecodePath, '-w', inputWitnessFile]; + const command = 'prove'; + const timer = new Timer(); + const logFunction = (message: string) => { + log(`${circuitName} BB out - ${message}`); + }; + const result = await executeBB(pathToBB, command, args, logFunction); + const duration = timer.ms(); + // cleanup the bytecode + await fs.rm(bytecodePath, { force: true }); + if (result == BB_RESULT.SUCCESS) { + return { status: BB_RESULT.SUCCESS, duration, path: outputPath }; + } + // Not a great error message here but it is difficult to decipher what comes from bb + return { status: BB_RESULT.FAILURE, reason: `Failed to generate proof` }; + } catch (error) { + return { status: BB_RESULT.FAILURE, reason: `${error}` }; + } +} + +/** + * Used for verifying proofs of noir circuits + * @param pathToBB - The full path to the bb binary + * @param proofFullPath - The full path to the proof to be verified + * @param verificationKeyPath - The full path to the circuit verification key + * @param log - A logging function + * @returns An object containing a result indication and duration taken + */ +export async function verifyProof( + pathToBB: string, + proofFullPath: string, + verificationKeyPath: string, + log: LogFn, +): Promise { + const binaryPresent = await fs + .access(pathToBB, fs.constants.R_OK) + .then(_ => true) + .catch(_ => false); + if (!binaryPresent) { + return { status: BB_RESULT.FAILURE, reason: `Failed to find bb binary at ${pathToBB}` }; + } + + try { + const args = ['-p', proofFullPath, '-k', verificationKeyPath]; + const timer = new Timer(); + const result = await executeBB(pathToBB, 'verify', args, log); + const duration = timer.ms(); + if (result == BB_RESULT.SUCCESS) { + return { status: BB_RESULT.SUCCESS, duration }; + } + // Not a great error message here but it is difficult to decipher what comes from bb + return { status: BB_RESULT.FAILURE, reason: `Failed to verify proof` }; + } catch (error) { + return { status: BB_RESULT.FAILURE, reason: `${error}` }; + } +} diff --git a/yarn-project/prover-client/src/bb/index.ts b/yarn-project/prover-client/src/bb/index.ts new file mode 100644 index 000000000000..9a19d22b7428 --- /dev/null +++ b/yarn-project/prover-client/src/bb/index.ts @@ -0,0 +1,23 @@ +#!/usr/bin/env -S node --no-warnings +import { createConsoleLogger } from '@aztec/foundation/log'; + +import 'source-map-support/register.js'; + +import { getProgram } from './cli.js'; + +const log = createConsoleLogger(); + +/** CLI main entrypoint */ +async function main() { + process.once('SIGINT', () => process.exit(0)); + process.once('SIGTERM', () => process.exit(0)); + + const program = getProgram(log); + await program.parseAsync(process.argv); +} + +main().catch(err => { + log(`Error in command execution`); + log(err); + process.exit(1); +}); diff --git a/yarn-project/prover-client/src/index.ts b/yarn-project/prover-client/src/index.ts index 46368c53575a..c47f1852f991 100644 --- a/yarn-project/prover-client/src/index.ts +++ b/yarn-project/prover-client/src/index.ts @@ -4,5 +4,4 @@ export * from './dummy-prover.js'; // Exported for integration_l1_publisher.test.ts export { getVerificationKeys } from './mocks/verification_keys.js'; -export { EmptyRollupProver } from './prover/empty.js'; export { RealRollupCircuitSimulator } from './simulator/rollup.js'; diff --git a/yarn-project/prover-client/src/mocks/fixtures.ts b/yarn-project/prover-client/src/mocks/fixtures.ts new file mode 100644 index 000000000000..6d691fe4ce7a --- /dev/null +++ b/yarn-project/prover-client/src/mocks/fixtures.ts @@ -0,0 +1,181 @@ +import { + MerkleTreeId, + type ProcessedTx, + makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, + makeProcessedTx, + mockTx, +} from '@aztec/circuit-types'; +import { + AztecAddress, + EthAddress, + Fr, + GasFees, + GlobalVariables, + KernelCircuitPublicInputs, + MAX_NEW_L2_TO_L1_MSGS_PER_TX, + MAX_NEW_NOTE_HASHES_PER_TX, + MAX_NEW_NULLIFIERS_PER_TX, + MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + NULLIFIER_TREE_HEIGHT, + PUBLIC_DATA_SUBTREE_HEIGHT, + PublicDataTreeLeaf, + PublicDataUpdateRequest, +} from '@aztec/circuits.js'; +import { fr, makeProof } from '@aztec/circuits.js/testing'; +import { makeTuple } from '@aztec/foundation/array'; +import { padArrayEnd } from '@aztec/foundation/collection'; +import { randomBytes } from '@aztec/foundation/crypto'; +import { type DebugLogger } from '@aztec/foundation/log'; +import { fileURLToPath } from '@aztec/foundation/url'; +import { NativeACVMSimulator, type SimulationProvider, WASMSimulator } from '@aztec/simulator'; +import { type MerkleTreeOperations } from '@aztec/world-state'; + +import * as fs from 'fs/promises'; +import path from 'path'; + +const { + BB_RELEASE_DIR = 'cpp/build/bin', + TEMP_DIR = '/tmp', + BB_BINARY_PATH = '', + BB_WORKING_DIRECTORY = '', + NOIR_RELEASE_DIR = 'noir-repo/target/release', + ACVM_BINARY_PATH = '', + ACVM_WORKING_DIRECTORY = '', +} = process.env; + +// Determines if we have access to the bb binary and a tmp folder for temp files +export const getConfig = async (logger: DebugLogger) => { + try { + const expectedBBPath = BB_BINARY_PATH + ? BB_BINARY_PATH + : `${path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../barretenberg/', BB_RELEASE_DIR)}/bb`; + await fs.access(expectedBBPath, fs.constants.R_OK); + const tempWorkingDirectory = `${TEMP_DIR}/${randomBytes(4).toString('hex')}`; + const bbWorkingDirectory = BB_WORKING_DIRECTORY ? BB_WORKING_DIRECTORY : `${tempWorkingDirectory}/bb`; + await fs.mkdir(bbWorkingDirectory, { recursive: true }); + logger.verbose(`Using native BB binary at ${expectedBBPath} with working directory ${bbWorkingDirectory}`); + + const expectedAcvmPath = ACVM_BINARY_PATH + ? ACVM_BINARY_PATH + : `${path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../noir/', NOIR_RELEASE_DIR)}/acvm`; + await fs.access(expectedAcvmPath, fs.constants.R_OK); + const acvmWorkingDirectory = ACVM_WORKING_DIRECTORY ? ACVM_WORKING_DIRECTORY : `${tempWorkingDirectory}/acvm`; + await fs.mkdir(acvmWorkingDirectory, { recursive: true }); + logger.verbose(`Using native ACVM binary at ${expectedAcvmPath} with working directory ${acvmWorkingDirectory}`); + return { + acvmWorkingDirectory, + bbWorkingDirectory, + expectedAcvmPath, + expectedBBPath, + directoryToCleanup: ACVM_WORKING_DIRECTORY && BB_WORKING_DIRECTORY ? undefined : tempWorkingDirectory, + }; + } catch (err) { + logger.verbose(`Native BB not available, error: ${err}`); + return undefined; + } +}; + +export async function getSimulationProvider( + config: { acvmWorkingDirectory: string | undefined; acvmBinaryPath: string | undefined }, + logger?: DebugLogger, +): Promise { + if (config.acvmBinaryPath && config.acvmWorkingDirectory) { + try { + await fs.access(config.acvmBinaryPath, fs.constants.R_OK); + await fs.mkdir(config.acvmWorkingDirectory, { recursive: true }); + logger?.info( + `Using native ACVM at ${config.acvmBinaryPath} and working directory ${config.acvmWorkingDirectory}`, + ); + return new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath); + } catch { + logger?.warn(`Failed to access ACVM at ${config.acvmBinaryPath}, falling back to WASM`); + } + } + logger?.info('Using WASM ACVM simulation'); + return new WASMSimulator(); +} + +export const makeBloatedProcessedTx = async (builderDb: MerkleTreeOperations, seed = 0x1) => { + seed *= MAX_NEW_NULLIFIERS_PER_TX; // Ensure no clashing given incremental seeds + const tx = mockTx(seed); + const kernelOutput = KernelCircuitPublicInputs.empty(); + kernelOutput.constants.historicalHeader = await builderDb.buildInitialHeader(); + kernelOutput.end.publicDataUpdateRequests = makeTuple( + MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + i => new PublicDataUpdateRequest(fr(i), fr(i + 10)), + seed + 0x500, + ); + kernelOutput.end.publicDataUpdateRequests = makeTuple( + MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + i => new PublicDataUpdateRequest(fr(i), fr(i + 10)), + seed + 0x600, + ); + + const processedTx = makeProcessedTx(tx, kernelOutput, makeProof(), []); + + processedTx.data.end.newNoteHashes = makeTuple(MAX_NEW_NOTE_HASHES_PER_TX, fr, seed + 0x100); + processedTx.data.end.newNullifiers = makeTuple(MAX_NEW_NULLIFIERS_PER_TX, fr, seed + 0x100000); + + processedTx.data.end.newNullifiers[tx.data.forPublic!.end.newNullifiers.length - 1] = Fr.zero(); + + processedTx.data.end.newL2ToL1Msgs = makeTuple(MAX_NEW_L2_TO_L1_MSGS_PER_TX, fr, seed + 0x300); + processedTx.data.end.encryptedLogsHash = Fr.fromBuffer(processedTx.encryptedLogs.hash()); + processedTx.data.end.unencryptedLogsHash = Fr.fromBuffer(processedTx.unencryptedLogs.hash()); + + return processedTx; +}; + +export const makeEmptyProcessedTx = async (builderDb: MerkleTreeOperations, chainId: Fr, version: Fr) => { + const header = await builderDb.buildInitialHeader(); + return makeEmptyProcessedTxFromHistoricalTreeRoots(header, chainId, version); +}; + +// Updates the expectedDb trees based on the new note hashes, contracts, and nullifiers from these txs +export const updateExpectedTreesFromTxs = async (db: MerkleTreeOperations, txs: ProcessedTx[]) => { + await db.appendLeaves( + MerkleTreeId.NOTE_HASH_TREE, + txs.flatMap(tx => + padArrayEnd( + tx.data.end.newNoteHashes.filter(x => !x.isZero()), + Fr.zero(), + MAX_NEW_NOTE_HASHES_PER_TX, + ), + ), + ); + await db.batchInsert( + MerkleTreeId.NULLIFIER_TREE, + txs.flatMap(tx => + padArrayEnd( + tx.data.end.newNullifiers.filter(x => !x.isZero()), + Fr.zero(), + MAX_NEW_NULLIFIERS_PER_TX, + ).map(x => x.toBuffer()), + ), + NULLIFIER_TREE_HEIGHT, + ); + for (const tx of txs) { + await db.batchInsert( + MerkleTreeId.PUBLIC_DATA_TREE, + tx.data.end.publicDataUpdateRequests.map(write => { + return new PublicDataTreeLeaf(write.leafSlot, write.newValue).toBuffer(); + }), + PUBLIC_DATA_SUBTREE_HEIGHT, + ); + } +}; + +export const makeGlobals = (blockNumber: number) => { + return new GlobalVariables( + Fr.ZERO, + Fr.ZERO, + new Fr(blockNumber), + Fr.ZERO, + EthAddress.ZERO, + AztecAddress.ZERO, + GasFees.empty(), + ); +}; + +export const makeEmptyProcessedTestTx = (builderDb: MerkleTreeOperations): Promise => { + return makeEmptyProcessedTx(builderDb, Fr.ZERO, Fr.ZERO); +}; diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index b8aeea5e45a2..fd727af7ed69 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -3,7 +3,6 @@ import { ARCHIVE_HEIGHT, AppendOnlyTreeSnapshot, type BaseOrMergeRollupPublicInputs, - type BaseParityInputs, BaseRollupInputs, ConstantRollupData, Fr, @@ -31,8 +30,7 @@ import { type PublicDataTreeLeafPreimage, ROLLUP_VK_TREE_HEIGHT, RollupTypes, - RootParityInput, - type RootParityInputs, + type RootParityInput, RootRollupInputs, type RootRollupPublicInputs, StateDiffHints, @@ -41,13 +39,10 @@ import { type VerificationKey, } from '@aztec/circuits.js'; import { assertPermutation, makeTuple } from '@aztec/foundation/array'; -import { type DebugLogger } from '@aztec/foundation/log'; import { type Tuple, assertLength, toFriendlyJSON } from '@aztec/foundation/serialize'; import { type MerkleTreeOperations } from '@aztec/world-state'; import { type VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; -import { type RollupProver } from '../prover/index.js'; -import { type RollupSimulator } from '../simulator/rollup.js'; // Denotes fields that are not used now, but will be in the future const FUTURE_FR = new Fr(0n); @@ -182,49 +177,6 @@ export function createMergeRollupInputs( return mergeInputs; } -export async function executeMergeRollupCircuit( - mergeInputs: MergeRollupInputs, - simulator: RollupSimulator, - prover: RollupProver, - logger?: DebugLogger, -): Promise<[BaseOrMergeRollupPublicInputs, Proof]> { - logger?.debug(`Running merge rollup circuit`); - const output = await simulator.mergeRollupCircuit(mergeInputs); - const proof = await prover.getMergeRollupProof(mergeInputs, output); - return [output, proof]; -} - -export async function executeRootRollupCircuit( - left: [BaseOrMergeRollupPublicInputs, Proof], - right: [BaseOrMergeRollupPublicInputs, Proof], - l1ToL2Roots: RootParityInput, - newL1ToL2Messages: Tuple, - messageTreeSnapshot: AppendOnlyTreeSnapshot, - messageTreeRootSiblingPath: Tuple, - simulator: RollupSimulator, - prover: RollupProver, - db: MerkleTreeOperations, - logger?: DebugLogger, -): Promise<[RootRollupPublicInputs, Proof]> { - logger?.debug(`Running root rollup circuit`); - const rootInput = await getRootRollupInput( - ...left, - ...right, - l1ToL2Roots, - newL1ToL2Messages, - messageTreeSnapshot, - messageTreeRootSiblingPath, - db, - ); - - // Simulate and get proof for the root circuit - const rootOutput = await simulator.rootRollupCircuit(rootInput); - - const rootProof = await prover.getRootRollupProof(rootInput, rootOutput); - - return [rootOutput, rootProof]; -} - // Validate that the roots of all local trees match the output of the root circuit simulation export async function validateRootOutput(rootOutput: RootRollupPublicInputs, db: MerkleTreeOperations) { await Promise.all([ @@ -448,21 +400,6 @@ export async function getMembershipWitnessFor( return new MembershipWitness(height, index, assertLength(path.toFields(), height)); } -export async function executeBaseRollupCircuit( - tx: ProcessedTx, - inputs: BaseRollupInputs, - treeSnapshots: Map, - simulator: RollupSimulator, - prover: RollupProver, - logger?: DebugLogger, -): Promise<[BaseOrMergeRollupPublicInputs, Proof]> { - logger?.debug(`Running base rollup for ${tx.hash}`); - const rollupOutput = await simulator.baseRollupCircuit(inputs); - validatePartialState(rollupOutput.end, treeSnapshots); - const proof = await prover.getBaseRollupProof(inputs, rollupOutput); - return [rollupOutput, proof]; -} - export function validatePartialState( partialState: PartialStateReference, treeSnapshots: Map, @@ -495,30 +432,6 @@ export function validateSimulatedTree( } } -export async function executeBaseParityCircuit( - inputs: BaseParityInputs, - simulator: RollupSimulator, - prover: RollupProver, - logger?: DebugLogger, -): Promise { - logger?.debug(`Running base parity circuit`); - const parityPublicInputs = await simulator.baseParityCircuit(inputs); - const proof = await prover.getBaseParityProof(inputs, parityPublicInputs); - return new RootParityInput(proof, parityPublicInputs); -} - -export async function executeRootParityCircuit( - inputs: RootParityInputs, - simulator: RollupSimulator, - prover: RollupProver, - logger?: DebugLogger, -): Promise { - logger?.debug(`Running root parity circuit`); - const parityPublicInputs = await simulator.rootParityCircuit(inputs); - const proof = await prover.getRootParityProof(inputs, parityPublicInputs); - return new RootParityInput(proof, parityPublicInputs); -} - export function validateTx(tx: ProcessedTx) { const txHeader = tx.data.constants.historicalHeader; if (txHeader.state.l1ToL2MessageTree.isZero()) { diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts deleted file mode 100644 index 05e78ec35890..000000000000 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts +++ /dev/null @@ -1,610 +0,0 @@ -import { - MerkleTreeId, - PROVING_STATUS, - type ProcessedTx, - type ProvingFailure, - makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, - makeProcessedTx, - mockTx, -} from '@aztec/circuit-types'; -import { - AztecAddress, - type BaseOrMergeRollupPublicInputs, - EthAddress, - Fr, - GasFees, - GlobalVariables, - KernelCircuitPublicInputs, - MAX_NEW_L2_TO_L1_MSGS_PER_TX, - MAX_NEW_NOTE_HASHES_PER_TX, - MAX_NEW_NULLIFIERS_PER_TX, - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - NULLIFIER_SUBTREE_HEIGHT, - NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, - PUBLIC_DATA_SUBTREE_HEIGHT, - Proof, - PublicDataTreeLeaf, - PublicDataUpdateRequest, - type RootRollupPublicInputs, -} from '@aztec/circuits.js'; -import { - fr, - makeBaseOrMergeRollupPublicInputs, - makeParityPublicInputs, - makeProof, - makeRootRollupPublicInputs, -} from '@aztec/circuits.js/testing'; -import { makeTuple, range } from '@aztec/foundation/array'; -import { padArrayEnd, times } from '@aztec/foundation/collection'; -import { sleep } from '@aztec/foundation/sleep'; -import { openTmpStore } from '@aztec/kv-store/utils'; -import { WASMSimulator } from '@aztec/simulator'; -import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; - -import { type MockProxy, mock } from 'jest-mock-extended'; -import { type MemDown, default as memdown } from 'memdown'; - -import { getVerificationKeys } from '../mocks/verification_keys.js'; -import { type RollupProver } from '../prover/index.js'; -import { type RollupSimulator } from '../simulator/rollup.js'; -import { ProvingOrchestrator } from './orchestrator.js'; - -export const createMemDown = () => (memdown as any)() as MemDown; - -describe('prover/tx-prover', () => { - let builder: ProvingOrchestrator; - let builderDb: MerkleTreeOperations; - let expectsDb: MerkleTreeOperations; - - let simulator: MockProxy; - let prover: MockProxy; - - let blockNumber: number; - let baseRollupOutputLeft: BaseOrMergeRollupPublicInputs; - let baseRollupOutputRight: BaseOrMergeRollupPublicInputs; - let rootRollupOutput: RootRollupPublicInputs; - let mockL1ToL2Messages: Fr[]; - - let globalVariables: GlobalVariables; - - const emptyProof = new Proof(Buffer.alloc(32, 0)); - - const chainId = Fr.ZERO; - const version = Fr.ZERO; - const coinbase = EthAddress.ZERO; - const feeRecipient = AztecAddress.ZERO; - - const makeGlobals = (blockNumber: number) => { - return new GlobalVariables(chainId, version, new Fr(blockNumber), Fr.ZERO, coinbase, feeRecipient, GasFees.empty()); - }; - - beforeEach(async () => { - blockNumber = 3; - globalVariables = makeGlobals(blockNumber); - - builderDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); - expectsDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); - simulator = mock(); - prover = mock(); - builder = new ProvingOrchestrator(builderDb, new WASMSimulator(), getVerificationKeys(), prover); - - // Create mock l1 to L2 messages - mockL1ToL2Messages = new Array(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP).fill(new Fr(0n)); - - // Create mock outputs for simulator - baseRollupOutputLeft = makeBaseOrMergeRollupPublicInputs(0, globalVariables); - baseRollupOutputRight = makeBaseOrMergeRollupPublicInputs(0, globalVariables); - rootRollupOutput = makeRootRollupPublicInputs(0); - rootRollupOutput.header.globalVariables = globalVariables; - - // Set up mocks - prover.getBaseParityProof.mockResolvedValue(emptyProof); - prover.getRootParityProof.mockResolvedValue(emptyProof); - prover.getBaseRollupProof.mockResolvedValue(emptyProof); - prover.getMergeRollupProof.mockResolvedValue(emptyProof); - prover.getRootRollupProof.mockResolvedValue(emptyProof); - simulator.baseParityCircuit - .mockResolvedValueOnce(makeParityPublicInputs(1)) - .mockResolvedValue(makeParityPublicInputs(2)) - .mockResolvedValue(makeParityPublicInputs(3)) - .mockResolvedValueOnce(makeParityPublicInputs(4)); - simulator.rootParityCircuit.mockResolvedValueOnce(makeParityPublicInputs(5)); - simulator.baseRollupCircuit - .mockResolvedValueOnce(baseRollupOutputLeft) - .mockResolvedValueOnce(baseRollupOutputRight); - simulator.rootRollupCircuit.mockResolvedValue(rootRollupOutput); - }, 20_000); - - const makeEmptyProcessedTx = async () => { - const header = await builderDb.buildInitialHeader(); - return makeEmptyProcessedTxFromHistoricalTreeRoots(header, chainId, version); - }; - - // Updates the expectedDb trees based on the new note hashes, contracts, and nullifiers from these txs - const updateExpectedTreesFromTxs = async (txs: ProcessedTx[]) => { - await expectsDb.appendLeaves( - MerkleTreeId.NOTE_HASH_TREE, - txs.flatMap(tx => - padArrayEnd( - tx.data.end.newNoteHashes.filter(x => !x.isZero()), - Fr.zero(), - MAX_NEW_NOTE_HASHES_PER_TX, - ), - ), - ); - await expectsDb.batchInsert( - MerkleTreeId.NULLIFIER_TREE, - txs.flatMap(tx => - padArrayEnd( - tx.data.end.newNullifiers.filter(x => !x.isZero()), - Fr.zero(), - MAX_NEW_NULLIFIERS_PER_TX, - ).map(x => x.toBuffer()), - ), - NULLIFIER_SUBTREE_HEIGHT, - ); - for (const tx of txs) { - await expectsDb.batchInsert( - MerkleTreeId.PUBLIC_DATA_TREE, - tx.data.end.publicDataUpdateRequests.map(write => { - return new PublicDataTreeLeaf(write.leafSlot, write.newValue).toBuffer(); - }), - PUBLIC_DATA_SUBTREE_HEIGHT, - ); - } - }; - - describe('error handling', () => { - beforeEach(async () => { - builder = await ProvingOrchestrator.new(builderDb, new WASMSimulator(), prover); - }); - - it.each([ - [ - 'Base Rollup Failed', - () => { - prover.getBaseRollupProof.mockRejectedValue('Base Rollup Failed'); - }, - ], - [ - 'Merge Rollup Failed', - () => { - prover.getMergeRollupProof.mockRejectedValue('Merge Rollup Failed'); - }, - ], - [ - 'Root Rollup Failed', - () => { - prover.getRootRollupProof.mockRejectedValue('Root Rollup Failed'); - }, - ], - [ - 'Base Parity Failed', - () => { - prover.getBaseParityProof.mockRejectedValue('Base Parity Failed'); - }, - ], - [ - 'Root Parity Failed', - () => { - prover.getRootParityProof.mockRejectedValue('Root Parity Failed'); - }, - ], - ] as const)( - 'handles a %s error', - async (message: string, fn: () => void) => { - fn(); - const txs = await Promise.all([ - makeEmptyProcessedTx(), - makeEmptyProcessedTx(), - makeEmptyProcessedTx(), - makeEmptyProcessedTx(), - ]); - - const blockTicket = await builder.startNewBlock(txs.length, globalVariables, [], await makeEmptyProcessedTx()); - - for (const tx of txs) { - await builder.addNewTx(tx); - } - await expect(blockTicket.provingPromise).resolves.toEqual({ status: PROVING_STATUS.FAILURE, reason: message }); - }, - 60000, - ); - - afterEach(async () => { - await builder.stop(); - }); - }); - - describe('circuits simulator', () => { - beforeEach(async () => { - builder = await ProvingOrchestrator.new(builderDb, new WASMSimulator(), prover); - }); - - afterEach(async () => { - await builder.stop(); - }); - - const makeBloatedProcessedTx = async (seed = 0x1) => { - seed *= MAX_NEW_NULLIFIERS_PER_TX; // Ensure no clashing given incremental seeds - const tx = mockTx(seed); - const kernelOutput = KernelCircuitPublicInputs.empty(); - kernelOutput.constants.historicalHeader = await builderDb.buildInitialHeader(); - kernelOutput.end.publicDataUpdateRequests = makeTuple( - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - i => new PublicDataUpdateRequest(fr(i), fr(i + 10)), - seed + 0x500, - ); - kernelOutput.end.publicDataUpdateRequests = makeTuple( - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - i => new PublicDataUpdateRequest(fr(i), fr(i + 10)), - seed + 0x600, - ); - - const processedTx = makeProcessedTx(tx, kernelOutput, makeProof()); - - processedTx.data.end.newNoteHashes = makeTuple(MAX_NEW_NOTE_HASHES_PER_TX, fr, seed + 0x100); - processedTx.data.end.newNullifiers = makeTuple(MAX_NEW_NULLIFIERS_PER_TX, fr, seed + 0x100000); - - processedTx.data.end.newNullifiers[tx.data.forPublic!.end.newNullifiers.length - 1] = Fr.zero(); - - processedTx.data.end.newL2ToL1Msgs = makeTuple(MAX_NEW_L2_TO_L1_MSGS_PER_TX, fr, seed + 0x300); - processedTx.data.end.encryptedLogsHash = Fr.fromBuffer(processedTx.encryptedLogs.hash()); - processedTx.data.end.unencryptedLogsHash = Fr.fromBuffer(processedTx.unencryptedLogs.hash()); - - return processedTx; - }; - - it.each([ - [0, 2], - [1, 2], - [4, 4], - [5, 8], - [9, 16], - ] as const)( - 'builds an L2 block with %i bloated txs and %i txs total', - async (bloatedCount: number, totalCount: number) => { - const noteHashTreeBefore = await builderDb.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); - const txs = [ - ...(await Promise.all(times(bloatedCount, makeBloatedProcessedTx))), - ...(await Promise.all(times(totalCount - bloatedCount, makeEmptyProcessedTx))), - ]; - - const blockTicket = await builder.startNewBlock( - txs.length, - globalVariables, - mockL1ToL2Messages, - await makeEmptyProcessedTx(), - ); - - for (const tx of txs) { - await builder.addNewTx(tx); - } - - const result = await blockTicket.provingPromise; - expect(result.status).toBe(PROVING_STATUS.SUCCESS); - - const finalisedBlock = await builder.finaliseBlock(); - - expect(finalisedBlock.block.number).toEqual(blockNumber); - - await updateExpectedTreesFromTxs(txs); - const noteHashTreeAfter = await builderDb.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); - - if (bloatedCount > 0) { - expect(noteHashTreeAfter.root).not.toEqual(noteHashTreeBefore.root); - } - - const expectedNoteHashTreeAfter = await expectsDb.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE).then(t => t.root); - expect(noteHashTreeAfter.root).toEqual(expectedNoteHashTreeAfter); - }, - 60000, - ); - - it('builds an empty L2 block', async () => { - const txs = await Promise.all([makeEmptyProcessedTx(), makeEmptyProcessedTx()]); - - const blockTicket = await builder.startNewBlock(txs.length, globalVariables, [], await makeEmptyProcessedTx()); - - for (const tx of txs) { - await builder.addNewTx(tx); - } - - const result = await blockTicket.provingPromise; - expect(result.status).toBe(PROVING_STATUS.SUCCESS); - const finalisedBlock = await builder.finaliseBlock(); - - expect(finalisedBlock.block.number).toEqual(blockNumber); - }, 30_000); - - it('builds a block with 1 transaction', async () => { - const txs = await Promise.all([makeEmptyProcessedTx()]); - - // This will need to be a 2 tx block - const blockTicket = await builder.startNewBlock(2, globalVariables, [], await makeEmptyProcessedTx()); - - for (const tx of txs) { - await builder.addNewTx(tx); - } - - // we need to complete the block as we have not added a full set of txs - await builder.setBlockCompleted(); - - const result = await blockTicket.provingPromise; - expect(result.status).toBe(PROVING_STATUS.SUCCESS); - const finalisedBlock = await builder.finaliseBlock(); - - expect(finalisedBlock.block.number).toEqual(blockNumber); - }, 30_000); - - it('builds multiple blocks in sequence', async () => { - const numBlocks = 5; - let header = await builderDb.buildInitialHeader(); - - for (let i = 0; i < numBlocks; i++) { - const tx = await makeBloatedProcessedTx(i + 1); - const emptyTx = await makeEmptyProcessedTx(); - tx.data.constants.historicalHeader = header; - emptyTx.data.constants.historicalHeader = header; - - const blockNum = i + 1000; - - const globals = makeGlobals(blockNum); - - // This will need to be a 2 tx block - const blockTicket = await builder.startNewBlock(2, globals, [], emptyTx); - - await builder.addNewTx(tx); - - // we need to complete the block as we have not added a full set of txs - await builder.setBlockCompleted(); - - const result = await blockTicket.provingPromise; - expect(result.status).toBe(PROVING_STATUS.SUCCESS); - const finalisedBlock = await builder.finaliseBlock(); - - expect(finalisedBlock.block.number).toEqual(blockNum); - header = finalisedBlock.block.header; - - await builderDb.commit(); - } - }, 60_000); - - it('builds a mixed L2 block', async () => { - const txs = await Promise.all([ - makeBloatedProcessedTx(1), - makeBloatedProcessedTx(2), - makeBloatedProcessedTx(3), - makeBloatedProcessedTx(4), - ]); - - const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); - - const blockTicket = await builder.startNewBlock( - txs.length, - globalVariables, - l1ToL2Messages, - await makeEmptyProcessedTx(), - ); - - for (const tx of txs) { - await builder.addNewTx(tx); - } - - const result = await blockTicket.provingPromise; - expect(result.status).toBe(PROVING_STATUS.SUCCESS); - const finalisedBlock = await builder.finaliseBlock(); - - expect(finalisedBlock.block.number).toEqual(blockNumber); - }, 200_000); - - it('builds a block concurrently with transactions', async () => { - const txs = await Promise.all([ - makeBloatedProcessedTx(1), - makeBloatedProcessedTx(2), - makeBloatedProcessedTx(3), - makeBloatedProcessedTx(4), - ]); - - const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); - - const blockTicket = await builder.startNewBlock( - txs.length, - globalVariables, - l1ToL2Messages, - await makeEmptyProcessedTx(), - ); - - for (const tx of txs) { - await builder.addNewTx(tx); - await sleep(1000); - } - - const result = await blockTicket.provingPromise; - expect(result.status).toBe(PROVING_STATUS.SUCCESS); - const finalisedBlock = await builder.finaliseBlock(); - - expect(finalisedBlock.block.number).toEqual(blockNumber); - }, 200_000); - - it('cancels current block and switches to new ones', async () => { - const txs1 = await Promise.all([makeBloatedProcessedTx(1), makeBloatedProcessedTx(2)]); - - const txs2 = await Promise.all([makeBloatedProcessedTx(3), makeBloatedProcessedTx(4)]); - - const globals1: GlobalVariables = makeGlobals(100); - const globals2: GlobalVariables = makeGlobals(101); - - const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); - - const blockTicket1 = await builder.startNewBlock(2, globals1, l1ToL2Messages, await makeEmptyProcessedTx()); - - await builder.addNewTx(txs1[0]); - await builder.addNewTx(txs1[1]); - - // Now we cancel the block. The first block will come to a stop as and when current proofs complete - builder.cancelBlock(); - - const result1 = await blockTicket1.provingPromise; - - // in all likelihood, the block will have a failure code as we cancelled it - // however it may have actually completed proving before we cancelled in which case it could be a succes code - if (result1.status === PROVING_STATUS.FAILURE) { - expect((result1 as ProvingFailure).reason).toBe('Proving cancelled'); - } - - await builderDb.rollback(); - - const blockTicket2 = await builder.startNewBlock(2, globals2, l1ToL2Messages, await makeEmptyProcessedTx()); - - await builder.addNewTx(txs2[0]); - await builder.addNewTx(txs2[1]); - - const result2 = await blockTicket2.provingPromise; - expect(result2.status).toBe(PROVING_STATUS.SUCCESS); - const finalisedBlock = await builder.finaliseBlock(); - - expect(finalisedBlock.block.number).toEqual(101); - }, 10000); - - it('automatically cancels an incomplete block when starting a new one', async () => { - const txs1 = await Promise.all([makeBloatedProcessedTx(1), makeBloatedProcessedTx(2)]); - - const txs2 = await Promise.all([makeBloatedProcessedTx(3), makeBloatedProcessedTx(4)]); - - const globals1: GlobalVariables = makeGlobals(100); - const globals2: GlobalVariables = makeGlobals(101); - - const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); - - const blockTicket1 = await builder.startNewBlock(2, globals1, l1ToL2Messages, await makeEmptyProcessedTx()); - - await builder.addNewTx(txs1[0]); - - await builderDb.rollback(); - - const blockTicket2 = await builder.startNewBlock(2, globals2, l1ToL2Messages, await makeEmptyProcessedTx()); - - await builder.addNewTx(txs2[0]); - await builder.addNewTx(txs2[1]); - - const result1 = await blockTicket1.provingPromise; - expect(result1.status).toBe(PROVING_STATUS.FAILURE); - expect((result1 as ProvingFailure).reason).toBe('Proving cancelled'); - - const result2 = await blockTicket2.provingPromise; - expect(result2.status).toBe(PROVING_STATUS.SUCCESS); - const finalisedBlock = await builder.finaliseBlock(); - - expect(finalisedBlock.block.number).toEqual(101); - }, 10000); - - it('builds an unbalanced L2 block', async () => { - const txs = await Promise.all([makeBloatedProcessedTx(1), makeBloatedProcessedTx(2), makeBloatedProcessedTx(3)]); - - const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); - - // this needs to be a 4 tx block that will need to be completed - const blockTicket = await builder.startNewBlock(4, globalVariables, l1ToL2Messages, await makeEmptyProcessedTx()); - - for (const tx of txs) { - await builder.addNewTx(tx); - } - - await builder.setBlockCompleted(); - - const result = await blockTicket.provingPromise; - expect(result.status).toBe(PROVING_STATUS.SUCCESS); - const finalisedBlock = await builder.finaliseBlock(); - - expect(finalisedBlock.block.number).toEqual(blockNumber); - }, 200_000); - - it('throws if adding too many transactions', async () => { - const txs = await Promise.all([ - makeBloatedProcessedTx(1), - makeBloatedProcessedTx(2), - makeBloatedProcessedTx(3), - makeBloatedProcessedTx(4), - ]); - - const blockTicket = await builder.startNewBlock(txs.length, globalVariables, [], await makeEmptyProcessedTx()); - - for (const tx of txs) { - await builder.addNewTx(tx); - } - - await expect(async () => await builder.addNewTx(await makeEmptyProcessedTx())).rejects.toThrow( - 'Rollup not accepting further transactions', - ); - - const result = await blockTicket.provingPromise; - expect(result.status).toBe(PROVING_STATUS.SUCCESS); - const finalisedBlock = await builder.finaliseBlock(); - - expect(finalisedBlock.block.number).toEqual(blockNumber); - }, 30_000); - - it('throws if adding a transaction before start', async () => { - await expect(async () => await builder.addNewTx(await makeEmptyProcessedTx())).rejects.toThrow( - `Invalid proving state, call startNewBlock before adding transactions`, - ); - }, 1000); - - it('throws if completing a block before start', async () => { - await expect(async () => await builder.setBlockCompleted()).rejects.toThrow( - 'Invalid proving state, call startNewBlock before adding transactions or completing the block', - ); - }, 1000); - - it('throws if finalising an incompletre block', async () => { - await expect(async () => await builder.finaliseBlock()).rejects.toThrow( - 'Invalid proving state, a block must be proven before it can be finalised', - ); - }, 1000); - - it('throws if finalising an already finalised block', async () => { - const txs = await Promise.all([makeEmptyProcessedTx(), makeEmptyProcessedTx()]); - - const blockTicket = await builder.startNewBlock(txs.length, globalVariables, [], await makeEmptyProcessedTx()); - - for (const tx of txs) { - await builder.addNewTx(tx); - } - - const result = await blockTicket.provingPromise; - expect(result.status).toBe(PROVING_STATUS.SUCCESS); - const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.block.number).toEqual(blockNumber); - await expect(async () => await builder.finaliseBlock()).rejects.toThrow('Block already finalised'); - }, 20000); - - it('throws if adding to a cancelled block', async () => { - await builder.startNewBlock(2, globalVariables, [], await makeEmptyProcessedTx()); - - builder.cancelBlock(); - - await expect(async () => await builder.addNewTx(await makeEmptyProcessedTx())).rejects.toThrow( - 'Rollup not accepting further transactions', - ); - }, 10000); - - it.each([[-4], [0], [1], [3], [8.1], [7]] as const)( - 'fails to start a block with %i transaxctions', - async (blockSize: number) => { - await expect( - async () => await builder.startNewBlock(blockSize, globalVariables, [], await makeEmptyProcessedTx()), - ).rejects.toThrow(`Length of txs for the block should be a power of two and at least two (got ${blockSize})`); - }, - 10000, - ); - - it('rejects if too many l1 to l2 messages are provided', async () => { - // Assemble a fake transaction - const l1ToL2Messages = new Array(100).fill(new Fr(0n)); - await expect( - async () => await builder.startNewBlock(2, globalVariables, l1ToL2Messages, await makeEmptyProcessedTx()), - ).rejects.toThrow('Too many L1 to L2 messages'); - }); - }); -}); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index cd5770e30b81..6090461179f0 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -18,7 +18,7 @@ import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NUM_BASE_PARITY_PER_ROOT_PARITY, type Proof, - type RootParityInput, + RootParityInput, RootParityInputs, } from '@aztec/circuits.js'; import { makeTuple } from '@aztec/foundation/array'; @@ -28,24 +28,18 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { type Tuple } from '@aztec/foundation/serialize'; import { sleep } from '@aztec/foundation/sleep'; import { elapsed } from '@aztec/foundation/timer'; -import { type SimulationProvider } from '@aztec/simulator'; import { type MerkleTreeOperations } from '@aztec/world-state'; import { inspect } from 'util'; -import { type VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; -import { type RollupProver } from '../prover/index.js'; -import { RealRollupCircuitSimulator, type RollupSimulator } from '../simulator/rollup.js'; +import { type CircuitProver } from '../prover/index.js'; import { buildBaseRollupInput, createMergeRollupInputs, - executeBaseParityCircuit, - executeBaseRollupCircuit, - executeMergeRollupCircuit, - executeRootParityCircuit, - executeRootRollupCircuit, + getRootRollupInput, getSubtreeSiblingPath, getTreeSnapshot, + validatePartialState, validateRootOutput, validateTx, } from './block-building-helpers.js'; @@ -82,6 +76,7 @@ export enum PROVING_JOB_TYPE { ROOT_ROLLUP, BASE_PARITY, ROOT_PARITY, + PUBLIC_KERNEL, } export type ProvingJob = { @@ -95,21 +90,16 @@ export type ProvingJob = { export class ProvingOrchestrator { private provingState: ProvingState | undefined = undefined; private jobQueue: MemoryFifo = new MemoryFifo(); - private simulator: RollupSimulator; private jobProcessPromise?: Promise; private stopped = false; constructor( private db: MerkleTreeOperations, - simulationProvider: SimulationProvider, - protected vks: VerificationKeys, - private prover: RollupProver, + private prover: CircuitProver, private maxConcurrentJobs = MAX_CONCURRENT_JOBS, - ) { - this.simulator = new RealRollupCircuitSimulator(simulationProvider); - } + ) {} - public static async new(db: MerkleTreeOperations, simulationProvider: SimulationProvider, prover: RollupProver) { - const orchestrator = new ProvingOrchestrator(db, simulationProvider, getVerificationKeys(), prover); + public static async new(db: MerkleTreeOperations, prover: CircuitProver) { + const orchestrator = new ProvingOrchestrator(db, prover); await orchestrator.start(); return Promise.resolve(orchestrator); } @@ -225,10 +215,11 @@ export class ProvingOrchestrator { logger.info(`Received transaction: ${tx.hash}`); // We start the transaction by enqueueing the state updates - const txIndex = this.provingState.addNewTx(tx); - // we start this transaction off by performing it's tree insertions and - await this.prepareBaseRollupInputs(this.provingState, BigInt(txIndex), tx); + await this.prepareBaseRollupInputs(this.provingState, tx); + this.enqueueJob(this.provingState, PROVING_JOB_TYPE.PUBLIC_KERNEL, () => + this.proveNextPublicFunction(this.provingState, txIndex, 0), + ); } /** @@ -247,7 +238,14 @@ export class ProvingOrchestrator { ); for (let i = this.provingState.transactionsReceived; i < this.provingState.totalNumTxs; i++) { const paddingTxIndex = this.provingState.addNewTx(this.provingState.emptyTx); - await this.prepareBaseRollupInputs(this.provingState, BigInt(paddingTxIndex), this.provingState!.emptyTx); + await this.prepareBaseRollupInputs(this.provingState, this.provingState!.emptyTx); + // TODO(@Phil): Properly encapsulate this stuff + const tx = this.provingState.allTxs[paddingTxIndex]; + const inputs = this.provingState.baseRollupInputs[paddingTxIndex]; + const treeSnapshots = this.provingState.txTreeSnapshots[paddingTxIndex]; + this.enqueueJob(this.provingState, PROVING_JOB_TYPE.BASE_ROLLUP, () => + this.runBaseRollup(this.provingState, BigInt(paddingTxIndex), tx, inputs, treeSnapshots), + ); } } @@ -338,8 +336,30 @@ export class ProvingOrchestrator { this.jobQueue.put(provingJob); } + private proveNextPublicFunction(provingState: ProvingState | undefined, txIndex: number, nextFunctionIndex: number) { + if (!provingState?.verifyState()) { + logger.debug(`Not executing public function, state invalid`); + return Promise.resolve(); + } + const request = provingState.getPublicFunction(txIndex, nextFunctionIndex); + if (!request) { + // TODO(@Phil): Properly encapsulate this stuff + const tx = provingState.allTxs[txIndex]; + const inputs = provingState.baseRollupInputs[txIndex]; + const treeSnapshots = provingState.txTreeSnapshots[txIndex]; + this.enqueueJob(provingState, PROVING_JOB_TYPE.BASE_ROLLUP, () => + this.runBaseRollup(provingState, BigInt(txIndex), tx, inputs, treeSnapshots), + ); + return Promise.resolve(); + } + this.enqueueJob(provingState, PROVING_JOB_TYPE.PUBLIC_KERNEL, () => + this.proveNextPublicFunction(provingState, txIndex, nextFunctionIndex + 1), + ); + return Promise.resolve(); + } + // Updates the merkle trees for a transaction. The first enqueued job for a transaction - private async prepareBaseRollupInputs(provingState: ProvingState | undefined, index: bigint, tx: ProcessedTx) { + private async prepareBaseRollupInputs(provingState: ProvingState | undefined, tx: ProcessedTx) { if (!provingState?.verifyState()) { logger.debug('Not preparing base rollup inputs, state invalid'); return; @@ -358,9 +378,9 @@ export class ProvingOrchestrator { logger.debug(`Discarding proving job, state no longer valid`); return; } - this.enqueueJob(provingState, PROVING_JOB_TYPE.BASE_ROLLUP, () => - this.runBaseRollup(provingState, index, tx, inputs, treeSnapshots), - ); + // TODO(@Phil): Properly encapsulate this stuff + provingState!.baseRollupInputs.push(inputs); + provingState!.txTreeSnapshots.push(treeSnapshots); } // Stores the intermediate inputs prepared for a merge proof @@ -392,15 +412,17 @@ export class ProvingOrchestrator { logger.debug('Not running base rollup, state invalid'); return; } - const [duration, baseRollupOutputs] = await elapsed(() => - executeBaseRollupCircuit(tx, inputs, treeSnapshots, this.simulator, this.prover, logger), - ); + const [duration, baseRollupOutputs] = await elapsed(async () => { + const [rollupOutput, proof] = await this.prover.getBaseRollupProof(inputs); + validatePartialState(rollupOutput.end, treeSnapshots); + return { rollupOutput, proof }; + }); logger.debug(`Simulated base rollup circuit`, { eventName: 'circuit-simulation', circuitName: 'base-rollup', duration, inputSize: inputs.toBuffer().length, - outputSize: baseRollupOutputs[0].toBuffer().length, + outputSize: baseRollupOutputs.rollupOutput.toBuffer().length, } satisfies CircuitSimulationStats); if (!provingState?.verifyState()) { logger.debug(`Discarding job as state no longer valid`); @@ -408,7 +430,10 @@ export class ProvingOrchestrator { } const currentLevel = provingState.numMergeLevels + 1n; logger.info(`Completed base rollup at index ${index}, current level ${currentLevel}`); - this.storeAndExecuteNextMergeLevel(provingState, currentLevel, index, baseRollupOutputs); + this.storeAndExecuteNextMergeLevel(provingState, currentLevel, index, [ + baseRollupOutputs.rollupOutput, + baseRollupOutputs.proof, + ]); } // Executes the merge rollup circuit and stored the output as intermediate state for the parent merge/root circuit @@ -427,9 +452,7 @@ export class ProvingOrchestrator { [mergeInputData.inputs[0]!, mergeInputData.proofs[0]!], [mergeInputData.inputs[1]!, mergeInputData.proofs[1]!], ); - const [duration, circuitOutputs] = await elapsed(() => - executeMergeRollupCircuit(circuitInputs, this.simulator, this.prover, logger), - ); + const [duration, circuitOutputs] = await elapsed(() => this.prover.getMergeRollupProof(circuitInputs)); logger.debug(`Simulated merge rollup circuit`, { eventName: 'circuit-simulation', circuitName: 'merge-rollup', @@ -453,22 +476,26 @@ export class ProvingOrchestrator { } const mergeInputData = provingState.getMergeInputs(0); const rootParityInput = provingState.finalRootParityInput!; - const [circuitsOutput, proof] = await executeRootRollupCircuit( - [mergeInputData.inputs[0]!, mergeInputData.proofs[0]!], - [mergeInputData.inputs[1]!, mergeInputData.proofs[1]!], + + const rootInput = await getRootRollupInput( + mergeInputData.inputs[0]!, + mergeInputData.proofs[0]!, + mergeInputData.inputs[1]!, + mergeInputData.proofs[1]!, rootParityInput, provingState.newL1ToL2Messages, provingState.messageTreeSnapshot, provingState.messageTreeRootSiblingPath, - this.simulator, - this.prover, this.db, - logger, ); + + // Simulate and get proof for the root circuit + const [rootOutput, rootProof] = await this.prover.getRootRollupProof(rootInput); + logger.info(`Completed root rollup`); - provingState.rootRollupPublicInputs = circuitsOutput; - provingState.finalProof = proof; + provingState.rootRollupPublicInputs = rootOutput; + provingState.finalProof = rootProof; const provingResult: ProvingResult = { status: PROVING_STATUS.SUCCESS, @@ -483,9 +510,10 @@ export class ProvingOrchestrator { logger.debug('Not running base parity, state no longer valid'); return; } - const [duration, circuitOutputs] = await elapsed(() => - executeBaseParityCircuit(inputs, this.simulator, this.prover, logger), - ); + const [duration, circuitOutputs] = await elapsed(async () => { + const [parityPublicInputs, proof] = await this.prover.getBaseParityProof(inputs); + return new RootParityInput(proof, parityPublicInputs); + }); logger.debug(`Simulated base parity circuit`, { eventName: 'circuit-simulation', circuitName: 'base-parity', @@ -519,9 +547,10 @@ export class ProvingOrchestrator { logger.debug(`Not running root parity circuit as state is no longer valid`); return; } - const [duration, circuitOutputs] = await elapsed(() => - executeRootParityCircuit(inputs, this.simulator, this.prover, logger), - ); + const [duration, circuitOutputs] = await elapsed(async () => { + const [parityPublicInputs, proof] = await this.prover.getRootParityProof(inputs); + return new RootParityInput(proof, parityPublicInputs); + }); logger.debug(`Simulated root parity circuit`, { eventName: 'circuit-simulation', circuitName: 'root-parity', diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts new file mode 100644 index 000000000000..1cd597e61a98 --- /dev/null +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts @@ -0,0 +1,155 @@ +import { PROVING_STATUS } from '@aztec/circuit-types'; +import { Fr, type GlobalVariables } from '@aztec/circuits.js'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; + +import { type MemDown, default as memdown } from 'memdown'; + +import { + getConfig, + getSimulationProvider, + makeBloatedProcessedTx, + makeEmptyProcessedTestTx, + makeGlobals, +} from '../mocks/fixtures.js'; +import { TestCircuitProver } from '../prover/test_circuit_prover.js'; +import { ProvingOrchestrator } from './orchestrator.js'; + +export const createMemDown = () => (memdown as any)() as MemDown; + +const logger = createDebugLogger('aztec:orchestrator-test'); + +describe('prover/orchestrator', () => { + let builder: ProvingOrchestrator; + let builderDb: MerkleTreeOperations; + + let prover: TestCircuitProver; + + let blockNumber: number; + + let globalVariables: GlobalVariables; + + beforeEach(async () => { + blockNumber = 3; + globalVariables = makeGlobals(blockNumber); + + const acvmConfig = await getConfig(logger); + const simulationProvider = await getSimulationProvider({ + acvmWorkingDirectory: acvmConfig?.acvmWorkingDirectory, + acvmBinaryPath: acvmConfig?.expectedAcvmPath, + }); + prover = new TestCircuitProver(simulationProvider); + + builderDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + builder = new ProvingOrchestrator(builderDb, prover, 1); + }, 20_000); + + describe('errors', () => { + beforeEach(async () => { + builder = await ProvingOrchestrator.new(builderDb, prover); + }); + + afterEach(async () => { + await builder.stop(); + }); + + it('throws if adding too many transactions', async () => { + const txs = await Promise.all([ + makeBloatedProcessedTx(builderDb, 1), + makeBloatedProcessedTx(builderDb, 2), + makeBloatedProcessedTx(builderDb, 3), + makeBloatedProcessedTx(builderDb, 4), + ]); + + const blockTicket = await builder.startNewBlock( + txs.length, + globalVariables, + [], + await makeEmptyProcessedTestTx(builderDb), + ); + + for (const tx of txs) { + await builder.addNewTx(tx); + } + + await expect(async () => await builder.addNewTx(await makeEmptyProcessedTestTx(builderDb))).rejects.toThrow( + 'Rollup not accepting further transactions', + ); + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.block.number).toEqual(blockNumber); + }, 30_000); + + it('throws if adding a transaction before start', async () => { + await expect(async () => await builder.addNewTx(await makeEmptyProcessedTestTx(builderDb))).rejects.toThrow( + `Invalid proving state, call startNewBlock before adding transactions`, + ); + }, 1000); + + it('throws if completing a block before start', async () => { + await expect(async () => await builder.setBlockCompleted()).rejects.toThrow( + 'Invalid proving state, call startNewBlock before adding transactions or completing the block', + ); + }, 1000); + + it('throws if finalising an incomplete block', async () => { + await expect(async () => await builder.finaliseBlock()).rejects.toThrow( + 'Invalid proving state, a block must be proven before it can be finalised', + ); + }, 1000); + + it('throws if finalising an already finalised block', async () => { + const txs = await Promise.all([makeEmptyProcessedTestTx(builderDb), makeEmptyProcessedTestTx(builderDb)]); + + const blockTicket = await builder.startNewBlock( + txs.length, + globalVariables, + [], + await makeEmptyProcessedTestTx(builderDb), + ); + + for (const tx of txs) { + await builder.addNewTx(tx); + } + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + expect(finalisedBlock.block.number).toEqual(blockNumber); + await expect(async () => await builder.finaliseBlock()).rejects.toThrow('Block already finalised'); + }, 60000); + + it('throws if adding to a cancelled block', async () => { + await builder.startNewBlock(2, globalVariables, [], await makeEmptyProcessedTestTx(builderDb)); + + builder.cancelBlock(); + + await expect(async () => await builder.addNewTx(await makeEmptyProcessedTestTx(builderDb))).rejects.toThrow( + 'Rollup not accepting further transactions', + ); + }, 10000); + + it.each([[-4], [0], [1], [3], [8.1], [7]] as const)( + 'fails to start a block with %i transactions', + async (blockSize: number) => { + await expect( + async () => + await builder.startNewBlock(blockSize, globalVariables, [], await makeEmptyProcessedTestTx(builderDb)), + ).rejects.toThrow(`Length of txs for the block should be a power of two and at least two (got ${blockSize})`); + }, + ); + + it('rejects if too many l1 to l2 messages are provided', async () => { + // Assemble a fake transaction + const l1ToL2Messages = new Array(100).fill(new Fr(0n)); + await expect( + async () => + await builder.startNewBlock(2, globalVariables, l1ToL2Messages, await makeEmptyProcessedTestTx(builderDb)), + ).rejects.toThrow('Too many L1 to L2 messages'); + }); + }); +}); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts new file mode 100644 index 000000000000..15c6210f44e2 --- /dev/null +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts @@ -0,0 +1,118 @@ +import { PROVING_STATUS, type ProcessedTx } from '@aztec/circuit-types'; +import { Fr, type GlobalVariables } from '@aztec/circuits.js'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { WASMSimulator } from '@aztec/simulator'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; + +import { jest } from '@jest/globals'; +import { type MemDown, default as memdown } from 'memdown'; + +import { getConfig, getSimulationProvider, makeEmptyProcessedTx, makeGlobals } from '../mocks/fixtures.js'; +import { type CircuitProver } from '../prover/index.js'; +import { TestCircuitProver } from '../prover/test_circuit_prover.js'; +import { ProvingOrchestrator } from './orchestrator.js'; + +export const createMemDown = () => (memdown as any)() as MemDown; + +const logger = createDebugLogger('aztec:orchestrator-test'); + +describe('prover/orchestrator', () => { + let builder: ProvingOrchestrator; + let builderDb: MerkleTreeOperations; + + let prover: TestCircuitProver; + + let blockNumber: number; + + let globalVariables: GlobalVariables; + + const makeEmptyProcessedTestTx = (): Promise => { + return makeEmptyProcessedTx(builderDb, Fr.ZERO, Fr.ZERO); + }; + + beforeEach(async () => { + blockNumber = 3; + globalVariables = makeGlobals(blockNumber); + + const acvmConfig = await getConfig(logger); + const simulationProvider = await getSimulationProvider({ + acvmWorkingDirectory: acvmConfig?.acvmWorkingDirectory, + acvmBinaryPath: acvmConfig?.expectedAcvmPath, + }); + prover = new TestCircuitProver(simulationProvider); + + builderDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + builder = new ProvingOrchestrator(builderDb, prover, 1); + }, 20_000); + + describe('error handling', () => { + let mockProver: CircuitProver; + + beforeEach(async () => { + mockProver = new TestCircuitProver(new WASMSimulator()); + builder = await ProvingOrchestrator.new(builderDb, mockProver); + }); + + it.each([ + [ + 'Base Rollup Failed', + () => { + jest.spyOn(mockProver, 'getBaseRollupProof').mockRejectedValue('Base Rollup Failed'); + }, + ], + [ + 'Merge Rollup Failed', + () => { + jest.spyOn(mockProver, 'getMergeRollupProof').mockRejectedValue('Merge Rollup Failed'); + }, + ], + [ + 'Root Rollup Failed', + () => { + jest.spyOn(mockProver, 'getRootRollupProof').mockRejectedValue('Root Rollup Failed'); + }, + ], + [ + 'Base Parity Failed', + () => { + jest.spyOn(mockProver, 'getBaseParityProof').mockRejectedValue('Base Parity Failed'); + }, + ], + [ + 'Root Parity Failed', + () => { + jest.spyOn(mockProver, 'getRootParityProof').mockRejectedValue('Root Parity Failed'); + }, + ], + ] as const)( + 'handles a %s error', + async (message: string, fn: () => void) => { + fn(); + const txs = await Promise.all([ + makeEmptyProcessedTestTx(), + makeEmptyProcessedTestTx(), + makeEmptyProcessedTestTx(), + makeEmptyProcessedTestTx(), + ]); + + const blockTicket = await builder.startNewBlock( + txs.length, + globalVariables, + [], + await makeEmptyProcessedTestTx(), + ); + + for (const tx of txs) { + await builder.addNewTx(tx); + } + await expect(blockTicket.provingPromise).resolves.toEqual({ status: PROVING_STATUS.FAILURE, reason: message }); + }, + 60000, + ); + + afterEach(async () => { + await builder.stop(); + }); + }); +}); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts new file mode 100644 index 000000000000..27108f89e674 --- /dev/null +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts @@ -0,0 +1,144 @@ +import { PROVING_STATUS, type ProvingFailure } from '@aztec/circuit-types'; +import { type GlobalVariables, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/circuits.js'; +import { fr } from '@aztec/circuits.js/testing'; +import { range } from '@aztec/foundation/array'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; + +import { type MemDown, default as memdown } from 'memdown'; + +import { + getConfig, + getSimulationProvider, + makeBloatedProcessedTx, + makeEmptyProcessedTestTx, + makeGlobals, +} from '../mocks/fixtures.js'; +import { TestCircuitProver } from '../prover/test_circuit_prover.js'; +import { ProvingOrchestrator } from './orchestrator.js'; + +export const createMemDown = () => (memdown as any)() as MemDown; + +const logger = createDebugLogger('aztec:orchestrator-test'); + +describe('prover/orchestrator', () => { + let builder: ProvingOrchestrator; + let builderDb: MerkleTreeOperations; + + let prover: TestCircuitProver; + + beforeEach(async () => { + const acvmConfig = await getConfig(logger); + const simulationProvider = await getSimulationProvider({ + acvmWorkingDirectory: acvmConfig?.acvmWorkingDirectory, + acvmBinaryPath: acvmConfig?.expectedAcvmPath, + }); + prover = new TestCircuitProver(simulationProvider); + + builderDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + builder = new ProvingOrchestrator(builderDb, prover, 1); + }, 20_000); + + describe('lifecycle', () => { + beforeEach(async () => { + builder = await ProvingOrchestrator.new(builderDb, prover); + }); + + afterEach(async () => { + await builder.stop(); + }); + + it('cancels current block and switches to new ones', async () => { + const txs1 = await Promise.all([makeBloatedProcessedTx(builderDb, 1), makeBloatedProcessedTx(builderDb, 2)]); + + const txs2 = await Promise.all([makeBloatedProcessedTx(builderDb, 3), makeBloatedProcessedTx(builderDb, 4)]); + + const globals1: GlobalVariables = makeGlobals(100); + const globals2: GlobalVariables = makeGlobals(101); + + const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); + + const blockTicket1 = await builder.startNewBlock( + 2, + globals1, + l1ToL2Messages, + await makeEmptyProcessedTestTx(builderDb), + ); + + await builder.addNewTx(txs1[0]); + await builder.addNewTx(txs1[1]); + + // Now we cancel the block. The first block will come to a stop as and when current proofs complete + builder.cancelBlock(); + + const result1 = await blockTicket1.provingPromise; + + // in all likelihood, the block will have a failure code as we cancelled it + // however it may have actually completed proving before we cancelled in which case it could be a success code + if (result1.status === PROVING_STATUS.FAILURE) { + expect((result1 as ProvingFailure).reason).toBe('Proving cancelled'); + } + + await builderDb.rollback(); + + const blockTicket2 = await builder.startNewBlock( + 2, + globals2, + l1ToL2Messages, + await makeEmptyProcessedTestTx(builderDb), + ); + + await builder.addNewTx(txs2[0]); + await builder.addNewTx(txs2[1]); + + const result2 = await blockTicket2.provingPromise; + expect(result2.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.block.number).toEqual(101); + }, 20000); + + it('automatically cancels an incomplete block when starting a new one', async () => { + const txs1 = await Promise.all([makeBloatedProcessedTx(builderDb, 1), makeBloatedProcessedTx(builderDb, 2)]); + + const txs2 = await Promise.all([makeBloatedProcessedTx(builderDb, 3), makeBloatedProcessedTx(builderDb, 4)]); + + const globals1: GlobalVariables = makeGlobals(100); + const globals2: GlobalVariables = makeGlobals(101); + + const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); + + const blockTicket1 = await builder.startNewBlock( + 2, + globals1, + l1ToL2Messages, + await makeEmptyProcessedTestTx(builderDb), + ); + + await builder.addNewTx(txs1[0]); + + await builderDb.rollback(); + + const blockTicket2 = await builder.startNewBlock( + 2, + globals2, + l1ToL2Messages, + await makeEmptyProcessedTestTx(builderDb), + ); + + await builder.addNewTx(txs2[0]); + await builder.addNewTx(txs2[1]); + + const result1 = await blockTicket1.provingPromise; + expect(result1.status).toBe(PROVING_STATUS.FAILURE); + expect((result1 as ProvingFailure).reason).toBe('Proving cancelled'); + + const result2 = await blockTicket2.provingPromise; + expect(result2.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.block.number).toEqual(101); + }, 20000); + }); +}); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks.test.ts new file mode 100644 index 000000000000..0277ae0664c8 --- /dev/null +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks.test.ts @@ -0,0 +1,89 @@ +import { PROVING_STATUS } from '@aztec/circuit-types'; +import { type GlobalVariables, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/circuits.js'; +import { fr } from '@aztec/circuits.js/testing'; +import { range } from '@aztec/foundation/array'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; + +import { type MemDown, default as memdown } from 'memdown'; + +import { + getConfig, + getSimulationProvider, + makeBloatedProcessedTx, + makeEmptyProcessedTestTx, + makeGlobals, +} from '../mocks/fixtures.js'; +import { TestCircuitProver } from '../prover/test_circuit_prover.js'; +import { ProvingOrchestrator } from './orchestrator.js'; + +export const createMemDown = () => (memdown as any)() as MemDown; + +const logger = createDebugLogger('aztec:orchestrator-test'); + +describe('prover/orchestrator', () => { + let builder: ProvingOrchestrator; + let builderDb: MerkleTreeOperations; + + let prover: TestCircuitProver; + + let blockNumber: number; + + let globalVariables: GlobalVariables; + + beforeEach(async () => { + blockNumber = 3; + globalVariables = makeGlobals(blockNumber); + + const acvmConfig = await getConfig(logger); + const simulationProvider = await getSimulationProvider({ + acvmWorkingDirectory: acvmConfig?.acvmWorkingDirectory, + acvmBinaryPath: acvmConfig?.expectedAcvmPath, + }); + prover = new TestCircuitProver(simulationProvider); + + builderDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + builder = new ProvingOrchestrator(builderDb, prover, 1); + }, 20_000); + + describe('blocks', () => { + beforeEach(async () => { + builder = await ProvingOrchestrator.new(builderDb, prover); + }); + + afterEach(async () => { + await builder.stop(); + }); + + it('builds an unbalanced L2 block', async () => { + const txs = await Promise.all([ + makeBloatedProcessedTx(builderDb, 1), + makeBloatedProcessedTx(builderDb, 2), + makeBloatedProcessedTx(builderDb, 3), + ]); + + const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); + + // this needs to be a 4 tx block that will need to be completed + const blockTicket = await builder.startNewBlock( + 4, + globalVariables, + l1ToL2Messages, + await makeEmptyProcessedTestTx(builderDb), + ); + + for (const tx of txs) { + await builder.addNewTx(tx); + } + + await builder.setBlockCompleted(); + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.block.number).toEqual(blockNumber); + }, 60_000); + }); +}); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts new file mode 100644 index 000000000000..353dd87fa9ac --- /dev/null +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts @@ -0,0 +1,111 @@ +import { MerkleTreeId, PROVING_STATUS } from '@aztec/circuit-types'; +import { type GlobalVariables, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/circuits.js'; +import { fr } from '@aztec/circuits.js/testing'; +import { range } from '@aztec/foundation/array'; +import { times } from '@aztec/foundation/collection'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; + +import { type MemDown, default as memdown } from 'memdown'; + +import { + getConfig, + getSimulationProvider, + makeBloatedProcessedTx, + makeEmptyProcessedTestTx, + makeGlobals, + updateExpectedTreesFromTxs, +} from '../mocks/fixtures.js'; +import { TestCircuitProver } from '../prover/test_circuit_prover.js'; +import { ProvingOrchestrator } from './orchestrator.js'; + +export const createMemDown = () => (memdown as any)() as MemDown; + +const logger = createDebugLogger('aztec:orchestrator-test'); + +describe('prover/orchestrator', () => { + let builder: ProvingOrchestrator; + let builderDb: MerkleTreeOperations; + let expectsDb: MerkleTreeOperations; + + let prover: TestCircuitProver; + + let blockNumber: number; + + let globalVariables: GlobalVariables; + + beforeEach(async () => { + blockNumber = 3; + globalVariables = makeGlobals(blockNumber); + + const acvmConfig = await getConfig(logger); + const simulationProvider = await getSimulationProvider({ + acvmWorkingDirectory: acvmConfig?.acvmWorkingDirectory, + acvmBinaryPath: acvmConfig?.expectedAcvmPath, + }); + prover = new TestCircuitProver(simulationProvider); + + builderDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + expectsDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + builder = new ProvingOrchestrator(builderDb, prover, 1); + }, 20_000); + + describe('blocks', () => { + beforeEach(async () => { + builder = await ProvingOrchestrator.new(builderDb, prover); + }); + + afterEach(async () => { + await builder.stop(); + }); + + it.each([ + [0, 2], + [1, 2], + [4, 4], + [5, 8], + [9, 16], + ] as const)( + 'builds an L2 block with %i bloated txs and %i txs total', + async (bloatedCount: number, totalCount: number) => { + const noteHashTreeBefore = await builderDb.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); + const txs = [ + ...(await Promise.all(times(bloatedCount, (i: number) => makeBloatedProcessedTx(builderDb, i)))), + ...(await Promise.all(times(totalCount - bloatedCount, _ => makeEmptyProcessedTestTx(builderDb)))), + ]; + + const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); + + const blockTicket = await builder.startNewBlock( + txs.length, + globalVariables, + l1ToL2Messages, + await makeEmptyProcessedTestTx(builderDb), + ); + + for (const tx of txs) { + await builder.addNewTx(tx); + } + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.block.number).toEqual(blockNumber); + + await updateExpectedTreesFromTxs(expectsDb, txs); + const noteHashTreeAfter = await builderDb.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); + + if (bloatedCount > 0) { + expect(noteHashTreeAfter.root).not.toEqual(noteHashTreeBefore.root); + } + + const expectedNoteHashTreeAfter = await expectsDb.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE).then(t => t.root); + expect(noteHashTreeAfter.root).toEqual(expectedNoteHashTreeAfter); + }, + 60000, + ); + }); +}); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_blocks.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_blocks.test.ts new file mode 100644 index 000000000000..3a2be210c63f --- /dev/null +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_blocks.test.ts @@ -0,0 +1,82 @@ +import { PROVING_STATUS } from '@aztec/circuit-types'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; + +import { type MemDown, default as memdown } from 'memdown'; + +import { + getConfig, + getSimulationProvider, + makeBloatedProcessedTx, + makeEmptyProcessedTestTx, + makeGlobals, +} from '../mocks/fixtures.js'; +import { TestCircuitProver } from '../prover/test_circuit_prover.js'; +import { ProvingOrchestrator } from './orchestrator.js'; + +export const createMemDown = () => (memdown as any)() as MemDown; + +const logger = createDebugLogger('aztec:orchestrator-test'); + +describe('prover/orchestrator', () => { + let builder: ProvingOrchestrator; + let builderDb: MerkleTreeOperations; + + let prover: TestCircuitProver; + + beforeEach(async () => { + const acvmConfig = await getConfig(logger); + const simulationProvider = await getSimulationProvider({ + acvmWorkingDirectory: acvmConfig?.acvmWorkingDirectory, + acvmBinaryPath: acvmConfig?.expectedAcvmPath, + }); + prover = new TestCircuitProver(simulationProvider); + + builderDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + builder = new ProvingOrchestrator(builderDb, prover, 1); + }, 20_000); + + describe('multiple blocks', () => { + beforeEach(async () => { + builder = await ProvingOrchestrator.new(builderDb, prover); + }); + + afterEach(async () => { + await builder.stop(); + }); + + it('builds multiple blocks in sequence', async () => { + const numBlocks = 5; + let header = await builderDb.buildInitialHeader(); + + for (let i = 0; i < numBlocks; i++) { + const tx = await makeBloatedProcessedTx(builderDb, i + 1); + const emptyTx = await makeEmptyProcessedTestTx(builderDb); + tx.data.constants.historicalHeader = header; + emptyTx.data.constants.historicalHeader = header; + + const blockNum = i + 1000; + + const globals = makeGlobals(blockNum); + + // This will need to be a 2 tx block + const blockTicket = await builder.startNewBlock(2, globals, [], emptyTx); + + await builder.addNewTx(tx); + + // we need to complete the block as we have not added a full set of txs + await builder.setBlockCompleted(); + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.block.number).toEqual(blockNum); + header = finalisedBlock.block.header; + + await builderDb.commit(); + } + }, 60_000); + }); +}); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts new file mode 100644 index 000000000000..87babf686e14 --- /dev/null +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts @@ -0,0 +1,189 @@ +import { PROVING_STATUS, type PublicKernelRequest, PublicKernelType } from '@aztec/circuit-types'; +import { type GlobalVariables, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/circuits.js'; +import { + fr, + makePublicKernelCircuitPrivateInputs, + makePublicKernelTailCircuitPrivateInputs, +} from '@aztec/circuits.js/testing'; +import { range } from '@aztec/foundation/array'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { sleep } from '@aztec/foundation/sleep'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; + +import { type MemDown, default as memdown } from 'memdown'; + +import { + getConfig, + getSimulationProvider, + makeBloatedProcessedTx, + makeEmptyProcessedTestTx, + makeGlobals, + updateExpectedTreesFromTxs, +} from '../mocks/fixtures.js'; +import { TestCircuitProver } from '../prover/test_circuit_prover.js'; +import { ProvingOrchestrator } from './orchestrator.js'; + +export const createMemDown = () => (memdown as any)() as MemDown; + +const logger = createDebugLogger('aztec:orchestrator-test'); + +describe('prover/orchestrator', () => { + let builder: ProvingOrchestrator; + let builderDb: MerkleTreeOperations; + let expectsDb: MerkleTreeOperations; + + let prover: TestCircuitProver; + + let blockNumber: number; + + let globalVariables: GlobalVariables; + + beforeEach(async () => { + blockNumber = 3; + globalVariables = makeGlobals(blockNumber); + + const acvmConfig = await getConfig(logger); + const simulationProvider = await getSimulationProvider({ + acvmWorkingDirectory: acvmConfig?.acvmWorkingDirectory, + acvmBinaryPath: acvmConfig?.expectedAcvmPath, + }); + prover = new TestCircuitProver(simulationProvider); + + builderDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + expectsDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + builder = new ProvingOrchestrator(builderDb, prover, 1); + }, 20_000); + + describe('blocks', () => { + beforeEach(async () => { + builder = await ProvingOrchestrator.new(builderDb, prover); + }); + + afterEach(async () => { + await builder.stop(); + }); + + it('builds an empty L2 block', async () => { + const txs = await Promise.all([makeEmptyProcessedTestTx(builderDb), makeEmptyProcessedTestTx(builderDb)]); + + const blockTicket = await builder.startNewBlock( + txs.length, + globalVariables, + [], + await makeEmptyProcessedTestTx(builderDb), + ); + + for (const tx of txs) { + await builder.addNewTx(tx); + } + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.block.number).toEqual(blockNumber); + }, 60_000); + + it('builds a block with 1 transaction', async () => { + const txs = await Promise.all([makeBloatedProcessedTx(builderDb, 1)]); + + await updateExpectedTreesFromTxs(expectsDb, txs); + + // This will need to be a 2 tx block + const blockTicket = await builder.startNewBlock( + 2, + globalVariables, + [], + await makeEmptyProcessedTestTx(builderDb), + ); + + for (const tx of txs) { + await builder.addNewTx(tx); + } + + // we need to complete the block as we have not added a full set of txs + await builder.setBlockCompleted(); + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.block.number).toEqual(blockNumber); + }, 60_000); + + it('builds a block with a transaction with public functions', async () => { + const tx = await makeBloatedProcessedTx(builderDb, 1); + + const setup: PublicKernelRequest = { + type: PublicKernelType.SETUP, + inputs: makePublicKernelCircuitPrivateInputs(2), + }; + + const app: PublicKernelRequest = { + type: PublicKernelType.APP_LOGIC, + inputs: makePublicKernelCircuitPrivateInputs(3), + }; + + const teardown: PublicKernelRequest = { + type: PublicKernelType.TEARDOWN, + inputs: makePublicKernelCircuitPrivateInputs(4), + }; + + const tail: PublicKernelRequest = { + type: PublicKernelType.TAIL, + inputs: makePublicKernelTailCircuitPrivateInputs(5), + }; + + tx.publicKernelRequests = [setup, app, teardown, tail]; + + // This will need to be a 2 tx block + const blockTicket = await builder.startNewBlock( + 2, + globalVariables, + [], + await makeEmptyProcessedTestTx(builderDb), + ); + + await builder.addNewTx(tx); + + // we need to complete the block as we have not added a full set of txs + await builder.setBlockCompleted(); + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.block.number).toEqual(blockNumber); + }, 60_000); + + it('builds a block concurrently with transaction simulation', async () => { + const txs = await Promise.all([ + makeBloatedProcessedTx(builderDb, 1), + makeBloatedProcessedTx(builderDb, 2), + makeBloatedProcessedTx(builderDb, 3), + makeBloatedProcessedTx(builderDb, 4), + ]); + + const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); + + const blockTicket = await builder.startNewBlock( + txs.length, + globalVariables, + l1ToL2Messages, + await makeEmptyProcessedTestTx(builderDb), + ); + + for (const tx of txs) { + await builder.addNewTx(tx); + await sleep(1000); + } + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.block.number).toEqual(blockNumber); + }, 60_000); + }); +}); diff --git a/yarn-project/prover-client/src/orchestrator/proving-state.ts b/yarn-project/prover-client/src/orchestrator/proving-state.ts index 4201de80a358..3f70a88b450f 100644 --- a/yarn-project/prover-client/src/orchestrator/proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/proving-state.ts @@ -1,7 +1,8 @@ -import { type L2Block, type ProcessedTx, type ProvingResult } from '@aztec/circuit-types'; +import { type L2Block, type MerkleTreeId, type ProcessedTx, type ProvingResult } from '@aztec/circuit-types'; import { type AppendOnlyTreeSnapshot, type BaseOrMergeRollupPublicInputs, + type BaseRollupInputs, type Fr, type GlobalVariables, type L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, @@ -30,7 +31,7 @@ enum PROVING_STATE_LIFECYCLE { * Captures resolve and reject callbacks to provide a promise base interface to the consumer of our proving. */ export class ProvingState { - private provingStateLifecyle = PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED; + private provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED; private mergeRollupInputs: MergeRollupInputData[] = []; private rootParityInputs: Array = []; private finalRootParityInputs: RootParityInput | undefined; @@ -38,6 +39,8 @@ export class ProvingState { public finalProof: Proof | undefined; public block: L2Block | undefined; private txs: ProcessedTx[] = []; + public baseRollupInputs: BaseRollupInputs[] = []; + public txTreeSnapshots: Map[] = []; constructor( public readonly totalNumTxs: number, private completionCallback: (result: ProvingResult) => void, @@ -63,7 +66,7 @@ export class ProvingState { public addNewTx(tx: ProcessedTx) { this.txs.push(tx); if (this.txs.length === this.totalNumTxs) { - this.provingStateLifecyle = PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL; + this.provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL; } return this.txs.length - 1; } @@ -86,13 +89,13 @@ export class ProvingState { public verifyState() { return ( - this.provingStateLifecyle === PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED || - this.provingStateLifecyle === PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL + this.provingStateLifecycle === PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED || + this.provingStateLifecycle === PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL ); } public isAcceptingTransactions() { - return this.provingStateLifecyle === PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED; + return this.provingStateLifecycle === PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED; } public get allTxs() { @@ -140,6 +143,21 @@ export class ProvingState { return this.rootParityInputs.findIndex(p => !p) === -1; } + public txHasPublicFunctions(index: number) { + return index >= 0 && this.txs.length > index && this.txs[index].publicKernelRequests.length; + } + + public getPublicFunction(txIndex: number, nextIndex: number) { + if (txIndex < 0 || txIndex >= this.txs.length) { + return undefined; + } + const tx = this.txs[txIndex]; + if (nextIndex < 0 || nextIndex >= tx.publicKernelRequests.length) { + return undefined; + } + return tx.publicKernelRequests[nextIndex]; + } + public cancel() { this.reject('Proving cancelled'); } @@ -148,7 +166,7 @@ export class ProvingState { if (!this.verifyState()) { return; } - this.provingStateLifecyle = PROVING_STATE_LIFECYCLE.PROVING_STATE_REJECTED; + this.provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_REJECTED; this.rejectionCallback(reason); } @@ -156,7 +174,7 @@ export class ProvingState { if (!this.verifyState()) { return; } - this.provingStateLifecyle = PROVING_STATE_LIFECYCLE.PROVING_STATE_RESOLVED; + this.provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_RESOLVED; this.completionCallback(result); } } diff --git a/yarn-project/prover-client/src/prover/bb_prover.test.ts b/yarn-project/prover-client/src/prover/bb_prover.test.ts new file mode 100644 index 000000000000..ad8ddd160b0f --- /dev/null +++ b/yarn-project/prover-client/src/prover/bb_prover.test.ts @@ -0,0 +1,105 @@ +import { PROVING_STATUS, makeEmptyProcessedTx } from '@aztec/circuit-types'; +import { Fr, type GlobalVariables, Header } from '@aztec/circuits.js'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; + +import * as fs from 'fs/promises'; +import { type MemDown, default as memdown } from 'memdown'; + +import { getConfig, makeBloatedProcessedTx, makeGlobals } from '../mocks/fixtures.js'; +import { buildBaseRollupInput } from '../orchestrator/block-building-helpers.js'; +import { ProvingOrchestrator } from '../orchestrator/orchestrator.js'; +import { BBNativeRollupProver, type BBProverConfig } from './bb_prover.js'; + +export const createMemDown = () => (memdown as any)() as MemDown; + +const logger = createDebugLogger('aztec:bb-prover-test'); + +describe('prover/bb_prover', () => { + let builderDb: MerkleTreeOperations; + let prover: BBNativeRollupProver; + let directoryToCleanup: string | undefined; + + let blockNumber: number; + + let globalVariables: GlobalVariables; + + beforeAll(async () => { + const config = await getConfig(logger); + if (!config) { + throw new Error(`BB and ACVM binaries must be present to test the BB Prover`); + } + directoryToCleanup = config.directoryToCleanup; + const bbConfig: BBProverConfig = { + acvmBinaryPath: config.expectedAcvmPath, + acvmWorkingDirectory: config.acvmWorkingDirectory, + bbBinaryPath: config.expectedBBPath, + bbWorkingDirectory: config.bbWorkingDirectory, + }; + prover = await BBNativeRollupProver.new(bbConfig); + }, 60_000); + + beforeEach(async () => { + blockNumber = 3; + globalVariables = makeGlobals(blockNumber); + + builderDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + }, 60_000); + + afterAll(async () => { + if (directoryToCleanup) { + await fs.rm(directoryToCleanup, { recursive: true, force: true }); + } + }, 5000); + + it('proves the base rollup', async () => { + const txs = await Promise.all([makeBloatedProcessedTx(builderDb, 1)]); + + logger.verbose('Building base rollup inputs'); + const baseRollupInputs = []; + for (const tx of txs) { + baseRollupInputs.push(await buildBaseRollupInput(tx, globalVariables, builderDb)); + } + logger.verbose('Proving base rollups'); + const proofOutputs = await Promise.all(baseRollupInputs.map(inputs => prover.getBaseRollupProof(inputs))); + logger.verbose('Verifying base rollups'); + await expect( + Promise.all(proofOutputs.map(output => prover.verifyProof('BaseRollupArtifact', output[1]))), + ).resolves.not.toThrow(); + }, 600_000); + + it('proves all circuits', async () => { + const txs = await Promise.all([ + makeBloatedProcessedTx(builderDb, 1), + makeBloatedProcessedTx(builderDb, 2), + makeBloatedProcessedTx(builderDb, 3), + makeBloatedProcessedTx(builderDb, 4), + ]); + + const orchestrator = await ProvingOrchestrator.new(builderDb, prover); + + const provingTicket = await orchestrator.startNewBlock( + 4, + globalVariables, + [], + makeEmptyProcessedTx(Header.empty(), new Fr(1234), new Fr(1)), + ); + + for (const tx of txs) { + await orchestrator.addNewTx(tx); + } + + await orchestrator.setBlockCompleted(); + + const provingResult = await provingTicket.provingPromise; + + expect(provingResult.status).toBe(PROVING_STATUS.SUCCESS); + + const blockResult = await orchestrator.finaliseBlock(); + + await expect(prover.verifyProof('RootRollupArtifact', blockResult.proof)).resolves.not.toThrow(); + + await orchestrator.stop(); + }, 600_000); +}); diff --git a/yarn-project/prover-client/src/prover/bb_prover.ts b/yarn-project/prover-client/src/prover/bb_prover.ts new file mode 100644 index 000000000000..29105d370ca3 --- /dev/null +++ b/yarn-project/prover-client/src/prover/bb_prover.ts @@ -0,0 +1,258 @@ +/* eslint-disable require-await */ +import { + type BaseOrMergeRollupPublicInputs, + type BaseParityInputs, + type BaseRollupInputs, + type MergeRollupInputs, + type ParityPublicInputs, + type PreviousRollupData, + Proof, + RollupTypes, + type RootParityInputs, + type RootRollupInputs, + type RootRollupPublicInputs, +} from '@aztec/circuits.js'; +import { randomBytes } from '@aztec/foundation/crypto'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { + ServerCircuitArtifacts, + type ServerProtocolArtifact, + convertBaseParityInputsToWitnessMap, + convertBaseParityOutputsFromWitnessMap, + convertBaseRollupInputsToWitnessMap, + convertBaseRollupOutputsFromWitnessMap, + convertMergeRollupInputsToWitnessMap, + convertMergeRollupOutputsFromWitnessMap, + convertRootParityInputsToWitnessMap, + convertRootParityOutputsFromWitnessMap, + convertRootRollupInputsToWitnessMap, + convertRootRollupOutputsFromWitnessMap, +} from '@aztec/noir-protocol-circuits-types'; +import { NativeACVMSimulator } from '@aztec/simulator'; + +import { type WitnessMap } from '@noir-lang/types'; +import * as fs from 'fs/promises'; + +import { BB_RESULT, generateKeyForNoirCircuit, generateProof, verifyProof } from '../bb/execute.js'; +import { type CircuitProver } from './interface.js'; + +const logger = createDebugLogger('aztec:bb-prover'); + +export type BBProverConfig = { + bbBinaryPath: string; + bbWorkingDirectory: string; + acvmBinaryPath: string; + acvmWorkingDirectory: string; +}; + +/** + * Prover implementation that uses barretenberg native proving + */ +export class BBNativeRollupProver implements CircuitProver { + private verificationKeyDirectories: Map = new Map(); + constructor(private config: BBProverConfig) {} + + static async new(config: BBProverConfig) { + await fs.access(config.acvmBinaryPath, fs.constants.R_OK); + await fs.mkdir(config.acvmWorkingDirectory, { recursive: true }); + await fs.access(config.bbBinaryPath, fs.constants.R_OK); + await fs.mkdir(config.bbWorkingDirectory, { recursive: true }); + logger.info(`Using native BB at ${config.bbBinaryPath} and working directory ${config.bbWorkingDirectory}`); + logger.info(`Using native ACVM at ${config.acvmBinaryPath} and working directory ${config.acvmWorkingDirectory}`); + + const prover = new BBNativeRollupProver(config); + await prover.init(); + return prover; + } + + /** + * Simulates the base parity circuit from its inputs. + * @param inputs - Inputs to the circuit. + * @returns The public inputs of the parity circuit. + */ + public async getBaseParityProof(inputs: BaseParityInputs): Promise<[ParityPublicInputs, Proof]> { + const witnessMap = convertBaseParityInputsToWitnessMap(inputs); + + const [outputWitness, proof] = await this.createProof(witnessMap, 'BaseParityArtifact'); + + const result = convertBaseParityOutputsFromWitnessMap(outputWitness); + + return Promise.resolve([result, proof]); + } + + /** + * Simulates the root parity circuit from its inputs. + * @param inputs - Inputs to the circuit. + * @returns The public inputs of the parity circuit. + */ + public async getRootParityProof(inputs: RootParityInputs): Promise<[ParityPublicInputs, Proof]> { + // verify all base parity inputs + await Promise.all(inputs.children.map(child => this.verifyProof('BaseParityArtifact', child.proof))); + + const witnessMap = convertRootParityInputsToWitnessMap(inputs); + + const [outputWitness, proof] = await this.createProof(witnessMap, 'RootParityArtifact'); + + const result = convertRootParityOutputsFromWitnessMap(outputWitness); + + return Promise.resolve([result, proof]); + } + + /** + * Simulates the base rollup circuit from its inputs. + * @param input - Inputs to the circuit. + * @returns The public inputs as outputs of the simulation. + */ + public async getBaseRollupProof(input: BaseRollupInputs): Promise<[BaseOrMergeRollupPublicInputs, Proof]> { + const witnessMap = convertBaseRollupInputsToWitnessMap(input); + + const [outputWitness, proof] = await this.createProof(witnessMap, 'BaseRollupArtifact'); + + const result = convertBaseRollupOutputsFromWitnessMap(outputWitness); + + return Promise.resolve([result, proof]); + } + /** + * Simulates the merge rollup circuit from its inputs. + * @param input - Inputs to the circuit. + * @returns The public inputs as outputs of the simulation. + */ + public async getMergeRollupProof(input: MergeRollupInputs): Promise<[BaseOrMergeRollupPublicInputs, Proof]> { + // verify both inputs + await Promise.all(input.previousRollupData.map(prev => this.verifyPreviousRollupProof(prev))); + + const witnessMap = convertMergeRollupInputsToWitnessMap(input); + + const [outputWitness, proof] = await this.createProof(witnessMap, 'MergeRollupArtifact'); + + const result = convertMergeRollupOutputsFromWitnessMap(outputWitness); + + return Promise.resolve([result, proof]); + } + + /** + * Simulates the root rollup circuit from its inputs. + * @param input - Inputs to the circuit. + * @returns The public inputs as outputs of the simulation. + */ + public async getRootRollupProof(input: RootRollupInputs): Promise<[RootRollupPublicInputs, Proof]> { + // verify the inputs + await Promise.all(input.previousRollupData.map(prev => this.verifyPreviousRollupProof(prev))); + + const witnessMap = convertRootRollupInputsToWitnessMap(input); + + const [outputWitness, proof] = await this.createProof(witnessMap, 'RootRollupArtifact'); + + await this.verifyProof('RootRollupArtifact', proof); + + const result = convertRootRollupOutputsFromWitnessMap(outputWitness); + return Promise.resolve([result, proof]); + } + + private async init() { + const promises = []; + for (const circuitName in ServerCircuitArtifacts) { + const verificationKeyPromise = generateKeyForNoirCircuit( + this.config.bbBinaryPath, + this.config.bbWorkingDirectory, + circuitName, + ServerCircuitArtifacts[circuitName as ServerProtocolArtifact], + 'vk', + logger.debug, + ).then(result => { + if (result.status == BB_RESULT.FAILURE) { + logger.error(`Failed to generate verification key for circuit ${circuitName}`); + return; + } + logger.info(`Generated verification key for circuit ${circuitName} at ${result.path!}`); + this.verificationKeyDirectories.set(circuitName as ServerProtocolArtifact, result.path!); + }); + promises.push(verificationKeyPromise); + } + await Promise.all(promises); + } + + public async createProof(witnessMap: WitnessMap, circuitType: ServerProtocolArtifact): Promise<[WitnessMap, Proof]> { + // Create random directory to be used for temp files + const bbWorkingDirectory = `${this.config.bbWorkingDirectory}/${randomBytes(8).toString('hex')}`; + await fs.mkdir(bbWorkingDirectory, { recursive: true }); + + await fs.access(bbWorkingDirectory); + + // Have the ACVM write the partial witness here + const outputWitnessFile = `${bbWorkingDirectory}/partial-witness.gz`; + + // Generate the partial witness using the ACVM + // A further temp directory will be created beneath ours and then cleaned up after the partial witness has been copied to our specified location + const simulator = new NativeACVMSimulator( + this.config.acvmWorkingDirectory, + this.config.acvmBinaryPath, + outputWitnessFile, + ); + + const artifact = ServerCircuitArtifacts[circuitType]; + + logger.debug(`Generating witness data for ${circuitType}`); + + const outputWitness = await simulator.simulateCircuit(witnessMap, artifact); + + // Now prove the circuit from the generated witness + logger.debug(`Proving ${circuitType}...`); + + const provingResult = await generateProof( + this.config.bbBinaryPath, + bbWorkingDirectory, + circuitType, + artifact, + outputWitnessFile, + logger.debug, + ); + + if (provingResult.status === BB_RESULT.FAILURE) { + logger.error(`Failed to generate proof for ${circuitType}: ${provingResult.reason}`); + throw new Error(provingResult.reason); + } + + // Read the proof and then cleanup up our temporary directory + const proofBuffer = await fs.readFile(provingResult.path!); + + await fs.rm(bbWorkingDirectory, { recursive: true, force: true }); + + logger.info( + `Generated proof for ${circuitType} in ${provingResult.duration} ms, size: ${proofBuffer.length} bytes`, + ); + + return [outputWitness, new Proof(proofBuffer)]; + } + + public async verifyProof(circuitType: ServerProtocolArtifact, proof: Proof) { + // Create random directory to be used for temp files + const bbWorkingDirectory = `${this.config.bbWorkingDirectory}/${randomBytes(8).toString('hex')}`; + await fs.mkdir(bbWorkingDirectory, { recursive: true }); + + const proofFileName = `${bbWorkingDirectory}/proof`; + const verificationKeyPath = this.verificationKeyDirectories.get(circuitType); + + await fs.writeFile(proofFileName, proof.buffer); + + const result = await verifyProof(this.config.bbBinaryPath, proofFileName, verificationKeyPath!, logger.debug); + + await fs.rm(bbWorkingDirectory, { recursive: true, force: true }); + + if (result.status === BB_RESULT.FAILURE) { + const errorMessage = `Failed to verify ${circuitType} proof!`; + throw new Error(errorMessage); + } + + logger.info(`Successfully verified ${circuitType} proof in ${result.duration} ms`); + } + + private async verifyPreviousRollupProof(previousRollupData: PreviousRollupData) { + const proof = previousRollupData.proof; + const circuitType = + previousRollupData.baseOrMergeRollupPublicInputs.rollupType === RollupTypes.Base + ? 'BaseRollupArtifact' + : 'MergeRollupArtifact'; + await this.verifyProof(circuitType, proof); + } +} diff --git a/yarn-project/prover-client/src/prover/empty.ts b/yarn-project/prover-client/src/prover/empty.ts deleted file mode 100644 index 31d20ebecc2b..000000000000 --- a/yarn-project/prover-client/src/prover/empty.ts +++ /dev/null @@ -1,102 +0,0 @@ -/* eslint-disable require-await */ -import { - AggregationObject, - type BaseOrMergeRollupPublicInputs, - type BaseParityInputs, - type BaseRollupInputs, - type KernelCircuitPublicInputs, - type MergeRollupInputs, - type ParityPublicInputs, - Proof, - type PublicCircuitPublicInputs, - type PublicKernelCircuitPublicInputs, - type RootParityInputs, - type RootRollupInputs, - type RootRollupPublicInputs, -} from '@aztec/circuits.js'; - -import { type PublicProver, type RollupProver } from './index.js'; - -const EMPTY_PROOF_SIZE = 42; - -// TODO: Silently modifying one of the inputs to inject the aggregation object is horrible. -// We should rethink these interfaces. - -/** - * Prover implementation that returns empty proofs and overrides aggregation objects. - */ -export class EmptyRollupProver implements RollupProver { - /** - * Creates an empty proof for the given input. - * @param inputs - Inputs to the circuit. - * @param publicInputs - Public inputs of the circuit obtained via simulation, modified by this call. - */ - async getBaseParityProof(inputs: BaseParityInputs, publicInputs: ParityPublicInputs): Promise { - publicInputs.aggregationObject = AggregationObject.makeFake(); - return new Proof(Buffer.alloc(EMPTY_PROOF_SIZE, 0)); - } - - /** - * Creates an empty proof for the given input. - * @param inputs - Inputs to the circuit. - * @param publicInputs - Public inputs of the circuit obtained via simulation, modified by this call. - */ - async getRootParityProof(inputs: RootParityInputs, publicInputs: ParityPublicInputs): Promise { - publicInputs.aggregationObject = AggregationObject.makeFake(); - return new Proof(Buffer.alloc(EMPTY_PROOF_SIZE, 0)); - } - - /** - * Creates an empty proof for the given input. - * @param _input - Input to the circuit. - * @param publicInputs - Public inputs of the circuit obtained via simulation, modified by this call. - */ - async getBaseRollupProof(_input: BaseRollupInputs, publicInputs: BaseOrMergeRollupPublicInputs): Promise { - publicInputs.aggregationObject = AggregationObject.makeFake(); - return new Proof(Buffer.alloc(EMPTY_PROOF_SIZE, 0)); - } - - /** - * Creates an empty proof for the given input. - * @param _input - Input to the circuit. - * @param publicInputs - Public inputs of the circuit obtained via simulation, modified by this call. - */ - async getMergeRollupProof(_input: MergeRollupInputs, publicInputs: BaseOrMergeRollupPublicInputs): Promise { - publicInputs.aggregationObject = AggregationObject.makeFake(); - return new Proof(Buffer.alloc(EMPTY_PROOF_SIZE, 0)); - } - /** - * Creates an empty proof for the given input. - * @param _input - Input to the circuit. - * @param publicInputs - Public inputs of the circuit obtained via simulation, modified by this call. - */ - async getRootRollupProof(_input: RootRollupInputs, publicInputs: RootRollupPublicInputs): Promise { - publicInputs.aggregationObject = AggregationObject.makeFake(); - return new Proof(Buffer.alloc(EMPTY_PROOF_SIZE, 0)); - } -} - -/** - * Prover implementation that returns empty proofs. - */ -export class EmptyPublicProver implements PublicProver { - /** - * Creates an empty proof for the given input. - * @param _publicInputs - Public inputs obtained via simulation. - */ - async getPublicCircuitProof(_publicInputs: PublicCircuitPublicInputs): Promise { - return new Proof(Buffer.alloc(EMPTY_PROOF_SIZE, 0)); - } - - /** - * Creates an empty proof for the given input. - * @param _publicInputs - Public inputs obtained via simulation. - */ - async getPublicKernelCircuitProof(_publicInputs: PublicKernelCircuitPublicInputs): Promise { - return new Proof(Buffer.alloc(EMPTY_PROOF_SIZE, 0)); - } - - async getPublicTailKernelCircuitProof(_publicInputs: KernelCircuitPublicInputs): Promise { - return new Proof(Buffer.alloc(EMPTY_PROOF_SIZE, 0)); - } -} diff --git a/yarn-project/prover-client/src/prover/index.ts b/yarn-project/prover-client/src/prover/index.ts index cb5e04129185..8a595f1c973c 100644 --- a/yarn-project/prover-client/src/prover/index.ts +++ b/yarn-project/prover-client/src/prover/index.ts @@ -1,77 +1 @@ -import { - type BaseOrMergeRollupPublicInputs, - type BaseParityInputs, - type BaseRollupInputs, - type KernelCircuitPublicInputs, - type MergeRollupInputs, - type ParityPublicInputs, - type Proof, - type PublicCircuitPublicInputs, - type PublicKernelCircuitPublicInputs, - type RootParityInputs, - type RootRollupInputs, - type RootRollupPublicInputs, -} from '@aztec/circuits.js'; - -/** - * Generates proofs for parity and rollup circuits. - */ -export interface RollupProver { - /** - * Creates a proof for the given input. - * @param input - Input to the circuit. - * @param publicInputs - Public inputs of the circuit obtained via simulation, modified by this call. - */ - getBaseParityProof(inputs: BaseParityInputs, publicInputs: ParityPublicInputs): Promise; - - /** - * Creates a proof for the given input. - * @param input - Input to the circuit. - * @param publicInputs - Public inputs of the circuit obtained via simulation, modified by this call. - */ - getRootParityProof(inputs: RootParityInputs, publicInputs: ParityPublicInputs): Promise; - - /** - * Creates a proof for the given input. - * @param input - Input to the circuit. - * @param publicInputs - Public inputs of the circuit obtained via simulation, modified by this call. - */ - getBaseRollupProof(input: BaseRollupInputs, publicInputs: BaseOrMergeRollupPublicInputs): Promise; - - /** - * Creates a proof for the given input. - * @param input - Input to the circuit. - * @param publicInputs - Public inputs of the circuit obtained via simulation, modified by this call. - */ - getMergeRollupProof(input: MergeRollupInputs, publicInputs: BaseOrMergeRollupPublicInputs): Promise; - - /** - * Creates a proof for the given input. - * @param input - Input to the circuit. - * @param publicInputs - Public inputs of the circuit obtained via simulation, modified by this call. - */ - getRootRollupProof(input: RootRollupInputs, publicInputs: RootRollupPublicInputs): Promise; -} - -/** - * Generates proofs for the public and public kernel circuits. - */ -export interface PublicProver { - /** - * Creates a proof for the given input. - * @param publicInputs - Public inputs obtained via simulation. - */ - getPublicCircuitProof(publicInputs: PublicCircuitPublicInputs): Promise; - - /** - * Creates a proof for the given input. - * @param publicInputs - Public inputs obtained via simulation. - */ - getPublicKernelCircuitProof(publicInputs: PublicKernelCircuitPublicInputs): Promise; - - /** - * Creates a proof for the given input. - * @param publicInputs - Public inputs obtained via simulation. - */ - getPublicTailKernelCircuitProof(publicInputs: KernelCircuitPublicInputs): Promise; -} +export * from './interface.js'; diff --git a/yarn-project/prover-client/src/prover/interface.ts b/yarn-project/prover-client/src/prover/interface.ts new file mode 100644 index 000000000000..2764f76dbf31 --- /dev/null +++ b/yarn-project/prover-client/src/prover/interface.ts @@ -0,0 +1,65 @@ +import { + type BaseOrMergeRollupPublicInputs, + type BaseParityInputs, + type BaseRollupInputs, + type MergeRollupInputs, + type ParityPublicInputs, + type Proof, + type PublicCircuitPublicInputs, + type PublicKernelCircuitPublicInputs, + type RootParityInputs, + type RootRollupInputs, + type RootRollupPublicInputs, +} from '@aztec/circuits.js'; + +/** + * Generates proofs for parity and rollup circuits. + */ +export interface CircuitProver { + /** + * Creates a proof for the given input. + * @param input - Input to the circuit. + */ + getBaseParityProof(inputs: BaseParityInputs): Promise<[ParityPublicInputs, Proof]>; + + /** + * Creates a proof for the given input. + * @param input - Input to the circuit. + */ + getRootParityProof(inputs: RootParityInputs): Promise<[ParityPublicInputs, Proof]>; + + /** + * Creates a proof for the given input. + * @param input - Input to the circuit. + */ + getBaseRollupProof(input: BaseRollupInputs): Promise<[BaseOrMergeRollupPublicInputs, Proof]>; + + /** + * Creates a proof for the given input. + * @param input - Input to the circuit. + */ + getMergeRollupProof(input: MergeRollupInputs): Promise<[BaseOrMergeRollupPublicInputs, Proof]>; + + /** + * Creates a proof for the given input. + * @param input - Input to the circuit. + */ + getRootRollupProof(input: RootRollupInputs): Promise<[RootRollupPublicInputs, Proof]>; +} + +/** + * Generates proofs for the public and public kernel circuits. + */ +export interface PublicProver { + /** + * Creates a proof for the given input. + * @param publicInputs - Public inputs obtained via simulation. + */ + getPublicCircuitProof(publicInputs: PublicCircuitPublicInputs): Promise; + + /** + * Creates a proof for the given input. + * @param publicInputs - Public inputs obtained via simulation. + */ + getPublicKernelCircuitProof(publicInputs: PublicKernelCircuitPublicInputs): Promise; +} diff --git a/yarn-project/prover-client/src/prover/test_circuit_prover.ts b/yarn-project/prover-client/src/prover/test_circuit_prover.ts new file mode 100644 index 000000000000..8be054a6e9ae --- /dev/null +++ b/yarn-project/prover-client/src/prover/test_circuit_prover.ts @@ -0,0 +1,133 @@ +import { type CircuitSimulationStats } from '@aztec/circuit-types/stats'; +import { + type BaseOrMergeRollupPublicInputs, + type BaseParityInputs, + type BaseRollupInputs, + type MergeRollupInputs, + type ParityPublicInputs, + type Proof, + type RootParityInputs, + type RootRollupInputs, + type RootRollupPublicInputs, + makeEmptyProof, +} from '@aztec/circuits.js'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { elapsed } from '@aztec/foundation/timer'; +import { + BaseParityArtifact, + MergeRollupArtifact, + RootParityArtifact, + RootRollupArtifact, + SimulatedBaseRollupArtifact, + convertBaseParityInputsToWitnessMap, + convertBaseParityOutputsFromWitnessMap, + convertMergeRollupInputsToWitnessMap, + convertMergeRollupOutputsFromWitnessMap, + convertRootParityInputsToWitnessMap, + convertRootParityOutputsFromWitnessMap, + convertRootRollupInputsToWitnessMap, + convertRootRollupOutputsFromWitnessMap, + convertSimulatedBaseRollupInputsToWitnessMap, + convertSimulatedBaseRollupOutputsFromWitnessMap, +} from '@aztec/noir-protocol-circuits-types'; +import { type SimulationProvider, WASMSimulator } from '@aztec/simulator'; + +import { type CircuitProver } from './interface.js'; + +/** + * A class for use in testing situations (e2e, unit test etc) + * Simulates circuits using the most efficient method and performs no proving + */ +export class TestCircuitProver implements CircuitProver { + private wasmSimulator = new WASMSimulator(); + + constructor( + private simulationProvider: SimulationProvider, + private logger = createDebugLogger('aztec:test-prover'), + ) {} + + /** + * Simulates the base parity circuit from its inputs. + * @param inputs - Inputs to the circuit. + * @returns The public inputs of the parity circuit. + */ + public async getBaseParityProof(inputs: BaseParityInputs): Promise<[ParityPublicInputs, Proof]> { + const witnessMap = convertBaseParityInputsToWitnessMap(inputs); + + // use WASM here as it is faster for small circuits + const witness = await this.wasmSimulator.simulateCircuit(witnessMap, BaseParityArtifact); + + const result = convertBaseParityOutputsFromWitnessMap(witness); + + return Promise.resolve([result, makeEmptyProof()]); + } + + /** + * Simulates the root parity circuit from its inputs. + * @param inputs - Inputs to the circuit. + * @returns The public inputs of the parity circuit. + */ + public async getRootParityProof(inputs: RootParityInputs): Promise<[ParityPublicInputs, Proof]> { + const witnessMap = convertRootParityInputsToWitnessMap(inputs); + + // use WASM here as it is faster for small circuits + const witness = await this.wasmSimulator.simulateCircuit(witnessMap, RootParityArtifact); + + const result = convertRootParityOutputsFromWitnessMap(witness); + + return Promise.resolve([result, makeEmptyProof()]); + } + + /** + * Simulates the base rollup circuit from its inputs. + * @param input - Inputs to the circuit. + * @returns The public inputs as outputs of the simulation. + */ + public async getBaseRollupProof(input: BaseRollupInputs): Promise<[BaseOrMergeRollupPublicInputs, Proof]> { + const witnessMap = convertSimulatedBaseRollupInputsToWitnessMap(input); + + const witness = await this.simulationProvider.simulateCircuit(witnessMap, SimulatedBaseRollupArtifact); + + const result = convertSimulatedBaseRollupOutputsFromWitnessMap(witness); + + return Promise.resolve([result, makeEmptyProof()]); + } + /** + * Simulates the merge rollup circuit from its inputs. + * @param input - Inputs to the circuit. + * @returns The public inputs as outputs of the simulation. + */ + public async getMergeRollupProof(input: MergeRollupInputs): Promise<[BaseOrMergeRollupPublicInputs, Proof]> { + const witnessMap = convertMergeRollupInputsToWitnessMap(input); + + // use WASM here as it is faster for small circuits + const witness = await this.wasmSimulator.simulateCircuit(witnessMap, MergeRollupArtifact); + + const result = convertMergeRollupOutputsFromWitnessMap(witness); + + return Promise.resolve([result, makeEmptyProof()]); + } + + /** + * Simulates the root rollup circuit from its inputs. + * @param input - Inputs to the circuit. + * @returns The public inputs as outputs of the simulation. + */ + public async getRootRollupProof(input: RootRollupInputs): Promise<[RootRollupPublicInputs, Proof]> { + const witnessMap = convertRootRollupInputsToWitnessMap(input); + + // use WASM here as it is faster for small circuits + const [duration, witness] = await elapsed(() => this.wasmSimulator.simulateCircuit(witnessMap, RootRollupArtifact)); + + const result = convertRootRollupOutputsFromWitnessMap(witness); + + this.logger.debug(`Simulated root rollup circuit`, { + eventName: 'circuit-simulation', + circuitName: 'root-rollup', + duration, + inputSize: input.toBuffer().length, + outputSize: result.toBuffer().length, + } satisfies CircuitSimulationStats); + return Promise.resolve([result, makeEmptyProof()]); + } +} diff --git a/yarn-project/prover-client/src/simulator/rollup.ts b/yarn-project/prover-client/src/simulator/rollup.ts index 624f7d41d19d..114873499a75 100644 --- a/yarn-project/prover-client/src/simulator/rollup.ts +++ b/yarn-project/prover-client/src/simulator/rollup.ts @@ -13,20 +13,20 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { elapsed } from '@aztec/foundation/timer'; import { BaseParityArtifact, - BaseRollupArtifact, MergeRollupArtifact, RootParityArtifact, RootRollupArtifact, + SimulatedBaseRollupArtifact, convertBaseParityInputsToWitnessMap, convertBaseParityOutputsFromWitnessMap, - convertBaseRollupInputsToWitnessMap, - convertBaseRollupOutputsFromWitnessMap, convertMergeRollupInputsToWitnessMap, convertMergeRollupOutputsFromWitnessMap, convertRootParityInputsToWitnessMap, convertRootParityOutputsFromWitnessMap, convertRootRollupInputsToWitnessMap, convertRootRollupOutputsFromWitnessMap, + convertSimulatedBaseRollupInputsToWitnessMap, + convertSimulatedBaseRollupOutputsFromWitnessMap, } from '@aztec/noir-protocol-circuits-types'; import { type SimulationProvider, WASMSimulator } from '@aztec/simulator'; @@ -113,11 +113,11 @@ export class RealRollupCircuitSimulator implements RollupSimulator { * @returns The public inputs as outputs of the simulation. */ public async baseRollupCircuit(input: BaseRollupInputs): Promise { - const witnessMap = convertBaseRollupInputsToWitnessMap(input); + const witnessMap = convertSimulatedBaseRollupInputsToWitnessMap(input); - const witness = await this.simulationProvider.simulateCircuit(witnessMap, BaseRollupArtifact); + const witness = await this.simulationProvider.simulateCircuit(witnessMap, SimulatedBaseRollupArtifact); - const result = convertBaseRollupOutputsFromWitnessMap(witness); + const result = convertSimulatedBaseRollupOutputsFromWitnessMap(witness); return Promise.resolve(result); } diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index 61b05ed416f9..daa7259f65ca 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -7,7 +7,7 @@ import { type WorldStateSynchronizer } from '@aztec/world-state'; import { type ProverConfig } from '../config.js'; import { type VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; import { ProvingOrchestrator } from '../orchestrator/orchestrator.js'; -import { EmptyRollupProver } from '../prover/empty.js'; +import { TestCircuitProver } from '../prover/test_circuit_prover.js'; /** * A prover accepting individual transaction requests @@ -21,9 +21,7 @@ export class TxProver implements ProverClient { ) { this.orchestrator = new ProvingOrchestrator( worldStateSynchronizer.getLatest(), - simulationProvider, - getVerificationKeys(), - new EmptyRollupProver(), + new TestCircuitProver(simulationProvider), ); } diff --git a/yarn-project/sequencer-client/src/sequencer/abstract_phase_manager.ts b/yarn-project/sequencer-client/src/sequencer/abstract_phase_manager.ts index ca12297b5faa..0733cb967e10 100644 --- a/yarn-project/sequencer-client/src/sequencer/abstract_phase_manager.ts +++ b/yarn-project/sequencer-client/src/sequencer/abstract_phase_manager.ts @@ -1,4 +1,10 @@ -import { MerkleTreeId, type SimulationError, type Tx, type UnencryptedFunctionL2Logs } from '@aztec/circuit-types'; +import { + MerkleTreeId, + type PublicKernelRequest, + type SimulationError, + type Tx, + type UnencryptedFunctionL2Logs, +} from '@aztec/circuit-types'; import { AztecAddress, CallRequest, @@ -103,6 +109,10 @@ export abstract class AbstractPhaseManager { publicKernelPublicInputs: PublicKernelCircuitPublicInputs, previousPublicKernelProof: Proof, ): Promise<{ + /** + * The collection of public kernel requests + */ + kernelRequests: PublicKernelRequest[]; /** * the output of the public kernel circuit for this phase */ @@ -201,6 +211,7 @@ export abstract class AbstractPhaseManager { previousPublicKernelProof: Proof, ): Promise< [ + PublicKernelCircuitPrivateInputs[], PublicKernelCircuitPublicInputs, Proof, UnencryptedFunctionL2Logs[], @@ -209,12 +220,13 @@ export abstract class AbstractPhaseManager { ] > { let kernelOutput = previousPublicKernelOutput; - let kernelProof = previousPublicKernelProof; + const kernelProof = previousPublicKernelProof; + const publicKernelInputs: PublicKernelCircuitPrivateInputs[] = []; const enqueuedCalls = this.extractEnqueuedPublicCalls(tx); if (!enqueuedCalls || !enqueuedCalls.length) { - return [kernelOutput, kernelProof, [], undefined, undefined]; + return [[], kernelOutput, kernelProof, [], undefined, undefined]; } const newUnencryptedFunctionLogs: UnencryptedFunctionL2Logs[] = []; @@ -260,7 +272,11 @@ export abstract class AbstractPhaseManager { executionStack.push(...result.nestedExecutions); const callData = await this.getPublicCallData(result, isExecutionRequest); - [kernelOutput, kernelProof] = await this.runKernelCircuit(kernelOutput, kernelProof, callData); + const circuitResult = await this.runKernelCircuit(kernelOutput, kernelProof, callData); + kernelOutput = circuitResult[1]; + + // Capture the inputs to the kernel circuit for later proving + publicKernelInputs.push(circuitResult[0]); // sanity check. Note we can't expect them to just be equal, because e.g. // if the simulator reverts in app logic, it "resets" and result.reverted will be false when we run teardown, @@ -279,7 +295,7 @@ export abstract class AbstractPhaseManager { result.revertReason }`, ); - return [kernelOutput, kernelProof, [], result.revertReason, undefined]; + return [[], kernelOutput, kernelProof, [], result.revertReason, undefined]; } if (!enqueuedExecutionResult) { @@ -304,33 +320,32 @@ export abstract class AbstractPhaseManager { // TODO(#3675): This should be done in a public kernel circuit removeRedundantPublicDataWrites(kernelOutput, this.phase); - return [kernelOutput, kernelProof, newUnencryptedFunctionLogs, undefined, returns]; + return [publicKernelInputs, kernelOutput, kernelProof, newUnencryptedFunctionLogs, undefined, returns]; } protected async runKernelCircuit( previousOutput: PublicKernelCircuitPublicInputs, previousProof: Proof, callData: PublicCallData, - ): Promise<[PublicKernelCircuitPublicInputs, Proof]> { - const output = await this.getKernelCircuitOutput(previousOutput, previousProof, callData); - return [output, makeEmptyProof()]; + ): Promise<[PublicKernelCircuitPrivateInputs, PublicKernelCircuitPublicInputs]> { + return await this.getKernelCircuitOutput(previousOutput, previousProof, callData); } - protected getKernelCircuitOutput( + protected async getKernelCircuitOutput( previousOutput: PublicKernelCircuitPublicInputs, previousProof: Proof, callData: PublicCallData, - ): Promise { + ): Promise<[PublicKernelCircuitPrivateInputs, PublicKernelCircuitPublicInputs]> { const previousKernel = this.getPreviousKernelData(previousOutput, previousProof); const inputs = new PublicKernelCircuitPrivateInputs(previousKernel, callData); switch (this.phase) { case PublicKernelPhase.SETUP: - return this.publicKernel.publicKernelCircuitSetup(inputs); + return [inputs, await this.publicKernel.publicKernelCircuitSetup(inputs)]; case PublicKernelPhase.APP_LOGIC: - return this.publicKernel.publicKernelCircuitAppLogic(inputs); + return [inputs, await this.publicKernel.publicKernelCircuitAppLogic(inputs)]; case PublicKernelPhase.TEARDOWN: - return this.publicKernel.publicKernelCircuitTeardown(inputs); + return [inputs, await this.publicKernel.publicKernelCircuitTeardown(inputs)]; default: throw new Error(`No public kernel circuit for inputs`); } diff --git a/yarn-project/sequencer-client/src/sequencer/app_logic_phase_manager.ts b/yarn-project/sequencer-client/src/sequencer/app_logic_phase_manager.ts index 06ff61023e31..831714e4468f 100644 --- a/yarn-project/sequencer-client/src/sequencer/app_logic_phase_manager.ts +++ b/yarn-project/sequencer-client/src/sequencer/app_logic_phase_manager.ts @@ -1,4 +1,4 @@ -import { type Tx } from '@aztec/circuit-types'; +import { type PublicKernelRequest, PublicKernelType, type Tx } from '@aztec/circuit-types'; import { type GlobalVariables, type Header, @@ -40,14 +40,20 @@ export class AppLogicPhaseManager extends AbstractPhaseManager { // TODO(@spalladino): Should we allow emitting contracts in the fee preparation phase? this.log.verbose(`Processing tx ${tx.getTxHash()}`); await this.publicContractsDB.addNewContracts(tx); - const [publicKernelOutput, publicKernelProof, newUnencryptedFunctionLogs, revertReason, returnValues] = - await this.processEnqueuedPublicCalls(tx, previousPublicKernelOutput, previousPublicKernelProof).catch( - // if we throw for any reason other than simulation, we need to rollback and drop the TX - async err => { - await this.publicStateDB.rollbackToCommit(); - throw err; - }, - ); + const [ + kernelInputs, + publicKernelOutput, + publicKernelProof, + newUnencryptedFunctionLogs, + revertReason, + returnValues, + ] = await this.processEnqueuedPublicCalls(tx, previousPublicKernelOutput, previousPublicKernelProof).catch( + // if we throw for any reason other than simulation, we need to rollback and drop the TX + async err => { + await this.publicStateDB.rollbackToCommit(); + throw err; + }, + ); if (revertReason) { await this.publicContractsDB.removeNewContracts(tx); @@ -57,6 +63,14 @@ export class AppLogicPhaseManager extends AbstractPhaseManager { await this.publicStateDB.checkpoint(); } - return { publicKernelOutput, publicKernelProof, revertReason, returnValues }; + // Return a list of app logic proving requests + const kernelRequests = kernelInputs.map(input => { + const request: PublicKernelRequest = { + type: PublicKernelType.APP_LOGIC, + inputs: input, + }; + return request; + }); + return { kernelRequests, publicKernelOutput, publicKernelProof, revertReason, returnValues }; } } diff --git a/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts b/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts index 567ec659e087..8abe3dc31b52 100644 --- a/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts @@ -116,6 +116,7 @@ describe('public_processor', () => { unencryptedLogs: tx.unencryptedLogs, isEmpty: false, revertReason: undefined, + publicKernelRequests: [], }; // Jest is complaining that the two objects are not equal, but they are. diff --git a/yarn-project/sequencer-client/src/sequencer/public_processor.ts b/yarn-project/sequencer-client/src/sequencer/public_processor.ts index ce68916309bc..e2df7c8f1abe 100644 --- a/yarn-project/sequencer-client/src/sequencer/public_processor.ts +++ b/yarn-project/sequencer-client/src/sequencer/public_processor.ts @@ -2,6 +2,7 @@ import { type BlockProver, type FailedTx, type ProcessedTx, + type PublicKernelRequest, type SimulationError, Tx, makeEmptyProcessedTx, @@ -105,7 +106,7 @@ export class PublicProcessor { } try { const [processedTx, returnValues] = !tx.hasPublicCalls() - ? [makeProcessedTx(tx, tx.data.toKernelCircuitPublicInputs(), tx.proof)] + ? [makeProcessedTx(tx, tx.data.toKernelCircuitPublicInputs(), tx.proof, [])] : await this.processTxWithPublicCalls(tx); validateProcessedTx(processedTx); // Re-validate the transaction @@ -151,6 +152,7 @@ export class PublicProcessor { private async processTxWithPublicCalls(tx: Tx): Promise<[ProcessedTx, ProcessReturnValues | undefined]> { let returnValues: ProcessReturnValues = undefined; + const publicRequests: PublicKernelRequest[] = []; let phase: AbstractPhaseManager | undefined = PhaseManagerFactory.phaseFromTx( tx, this.db, @@ -172,6 +174,7 @@ export class PublicProcessor { if (phase.phase === PublicKernelPhase.APP_LOGIC) { returnValues = output.returnValues; } + publicRequests.push(...output.kernelRequests); publicKernelPublicInput = output.publicKernelOutput; finalKernelOutput = output.finalKernelOutput; proof = output.publicKernelProof; @@ -193,7 +196,7 @@ export class PublicProcessor { throw new Error('Final public kernel was not executed.'); } - const processedTx = makeProcessedTx(tx, finalKernelOutput, proof, revertReason); + const processedTx = makeProcessedTx(tx, finalKernelOutput, proof, publicRequests, revertReason); this.log.debug(`Processed public part of ${tx.getTxHash()}`, { eventName: 'tx-sequencer-processing', diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index e47ba919dafc..b3e2ab97bdcd 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -75,7 +75,7 @@ describe('sequencer', () => { publicProcessor = mock({ process: async txs => [ - await Promise.all(txs.map(tx => makeProcessedTx(tx, tx.data.toKernelCircuitPublicInputs(), makeProof()))), + await Promise.all(txs.map(tx => makeProcessedTx(tx, tx.data.toKernelCircuitPublicInputs(), makeProof(), []))), [], [], ], diff --git a/yarn-project/sequencer-client/src/sequencer/setup_phase_manager.ts b/yarn-project/sequencer-client/src/sequencer/setup_phase_manager.ts index a043cfdd47b6..d1735e525006 100644 --- a/yarn-project/sequencer-client/src/sequencer/setup_phase_manager.ts +++ b/yarn-project/sequencer-client/src/sequencer/setup_phase_manager.ts @@ -1,4 +1,4 @@ -import { type Tx } from '@aztec/circuit-types'; +import { type PublicKernelRequest, PublicKernelType, type Tx } from '@aztec/circuit-types'; import { type GlobalVariables, type Header, @@ -35,7 +35,7 @@ export class SetupPhaseManager extends AbstractPhaseManager { previousPublicKernelProof: Proof, ) { this.log.verbose(`Processing tx ${tx.getTxHash()}`); - const [publicKernelOutput, publicKernelProof, newUnencryptedFunctionLogs, revertReason] = + const [kernelInputs, publicKernelOutput, publicKernelProof, newUnencryptedFunctionLogs, revertReason] = await this.processEnqueuedPublicCalls(tx, previousPublicKernelOutput, previousPublicKernelProof).catch( // the abstract phase manager throws if simulation gives error in a non-revertible phase async err => { @@ -45,6 +45,22 @@ export class SetupPhaseManager extends AbstractPhaseManager { ); tx.unencryptedLogs.addFunctionLogs(newUnencryptedFunctionLogs); await this.publicStateDB.checkpoint(); - return { publicKernelOutput, publicKernelProof, revertReason, returnValues: undefined }; + + // Return a list of setup proving requests + const kernelRequests = kernelInputs.map(input => { + const request: PublicKernelRequest = { + type: PublicKernelType.SETUP, + inputs: input, + }; + return request; + }); + return { + kernelRequests, + kernelInputs, + publicKernelOutput, + publicKernelProof, + revertReason, + returnValues: undefined, + }; } } diff --git a/yarn-project/sequencer-client/src/sequencer/tail_phase_manager.ts b/yarn-project/sequencer-client/src/sequencer/tail_phase_manager.ts index d1859dd0167b..87fbf9b460e9 100644 --- a/yarn-project/sequencer-client/src/sequencer/tail_phase_manager.ts +++ b/yarn-project/sequencer-client/src/sequencer/tail_phase_manager.ts @@ -1,4 +1,4 @@ -import { type Tx } from '@aztec/circuit-types'; +import { type PublicKernelRequest, PublicKernelType, type Tx } from '@aztec/circuit-types'; import { type Fr, type GlobalVariables, @@ -37,7 +37,7 @@ export class TailPhaseManager extends AbstractPhaseManager { async handle(tx: Tx, previousPublicKernelOutput: PublicKernelCircuitPublicInputs, previousPublicKernelProof: Proof) { this.log.verbose(`Processing tx ${tx.getTxHash()}`); - const [finalKernelOutput, publicKernelProof] = await this.runTailKernelCircuit( + const [inputs, finalKernelOutput] = await this.runTailKernelCircuit( previousPublicKernelOutput, previousPublicKernelProof, ).catch( @@ -51,10 +51,17 @@ export class TailPhaseManager extends AbstractPhaseManager { // commit the state updates from this transaction await this.publicStateDB.commit(); + // Return a tail proving request + const request: PublicKernelRequest = { + type: PublicKernelType.TAIL, + inputs: inputs, + }; + return { + kernelRequests: [request], publicKernelOutput: previousPublicKernelOutput, finalKernelOutput, - publicKernelProof, + publicKernelProof: makeEmptyProof(), revertReason: undefined, returnValues: undefined, }; @@ -63,8 +70,8 @@ export class TailPhaseManager extends AbstractPhaseManager { private async runTailKernelCircuit( previousOutput: PublicKernelCircuitPublicInputs, previousProof: Proof, - ): Promise<[KernelCircuitPublicInputs, Proof]> { - const output = await this.simulate(previousOutput, previousProof); + ): Promise<[PublicKernelTailCircuitPrivateInputs, KernelCircuitPublicInputs]> { + const [inputs, output] = await this.simulate(previousOutput, previousProof); // Temporary hack. Should sort them in the tail circuit. const noteHashes = mergeAccumulatedData( @@ -74,13 +81,13 @@ export class TailPhaseManager extends AbstractPhaseManager { ); output.end.newNoteHashes = this.sortNoteHashes(noteHashes); - return [output, makeEmptyProof()]; + return [inputs, output]; } private async simulate( previousOutput: PublicKernelCircuitPublicInputs, previousProof: Proof, - ): Promise { + ): Promise<[PublicKernelTailCircuitPrivateInputs, KernelCircuitPublicInputs]> { const previousKernel = this.getPreviousKernelData(previousOutput, previousProof); const { validationRequests, endNonRevertibleData, end } = previousOutput; @@ -99,7 +106,7 @@ export class TailPhaseManager extends AbstractPhaseManager { nullifierReadRequestHints, nullifierNonExistentReadRequestHints, ); - return this.publicKernel.publicKernelCircuitTail(inputs); + return [inputs, await this.publicKernel.publicKernelCircuitTail(inputs)]; } private sortNoteHashes(noteHashes: Tuple): Tuple { diff --git a/yarn-project/sequencer-client/src/sequencer/teardown_phase_manager.ts b/yarn-project/sequencer-client/src/sequencer/teardown_phase_manager.ts index fb6820fc7999..0fd37bb93404 100644 --- a/yarn-project/sequencer-client/src/sequencer/teardown_phase_manager.ts +++ b/yarn-project/sequencer-client/src/sequencer/teardown_phase_manager.ts @@ -1,4 +1,4 @@ -import { type Tx } from '@aztec/circuit-types'; +import { type PublicKernelRequest, PublicKernelType, type Tx } from '@aztec/circuit-types'; import { type GlobalVariables, type Header, @@ -35,7 +35,7 @@ export class TeardownPhaseManager extends AbstractPhaseManager { previousPublicKernelProof: Proof, ) { this.log.verbose(`Processing tx ${tx.getTxHash()}`); - const [publicKernelOutput, publicKernelProof, newUnencryptedFunctionLogs, revertReason] = + const [kernelInputs, publicKernelOutput, publicKernelProof, newUnencryptedFunctionLogs, revertReason] = await this.processEnqueuedPublicCalls(tx, previousPublicKernelOutput, previousPublicKernelProof).catch( // the abstract phase manager throws if simulation gives error in a non-revertible phase async err => { @@ -45,6 +45,22 @@ export class TeardownPhaseManager extends AbstractPhaseManager { ); tx.unencryptedLogs.addFunctionLogs(newUnencryptedFunctionLogs); await this.publicStateDB.checkpoint(); - return { publicKernelOutput, publicKernelProof, revertReason, returnValues: undefined }; + + // Return a list of teardown proving requests + const kernelRequests = kernelInputs.map(input => { + const request: PublicKernelRequest = { + type: PublicKernelType.TEARDOWN, + inputs: input, + }; + return request; + }); + return { + kernelRequests, + kernelInputs, + publicKernelOutput, + publicKernelProof, + revertReason, + returnValues: undefined, + }; } } diff --git a/yarn-project/simulator/src/simulator/acvm_native.ts b/yarn-project/simulator/src/simulator/acvm_native.ts index df1df4a1be63..070e77ad114c 100644 --- a/yarn-project/simulator/src/simulator/acvm_native.ts +++ b/yarn-project/simulator/src/simulator/acvm_native.ts @@ -1,4 +1,6 @@ import { randomBytes } from '@aztec/foundation/crypto'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { Timer } from '@aztec/foundation/timer'; import { type NoirCompiledCircuit } from '@aztec/types/noir'; import { type WitnessMap } from '@noir-lang/types'; @@ -7,6 +9,26 @@ import fs from 'fs/promises'; import { type SimulationProvider } from './simulation_provider.js'; +const logger = createDebugLogger('aztec:acvm-native'); + +export enum ACVM_RESULT { + SUCCESS, + FAILURE, +} + +export type ACVMSuccess = { + status: ACVM_RESULT.SUCCESS; + duration: number; + witness: Map; +}; + +export type ACVMFailure = { + status: ACVM_RESULT.FAILURE; + reason: string; +}; + +export type ACVMResult = ACVMSuccess | ACVMFailure; + /** * Parses a TOML format witness map string into a Map structure * @param outputString - The witness map in TOML format @@ -29,7 +51,8 @@ function parseIntoWitnessMap(outputString: string) { * @param inputWitness - The circuit's input witness * @param bytecode - The circuit bytecode * @param workingDirectory - A directory to use for temporary files by the ACVM - * @param pathToAcvm - The path to the ACVm binary + * @param pathToAcvm - The path to the ACVM binary + * @param outputFilename - If specified, the output will be stored as a file, encoded using Bincode * @returns The completed partial witness outputted from the circuit */ export async function executeNativeCircuit( @@ -37,7 +60,8 @@ export async function executeNativeCircuit( bytecode: Buffer, workingDirectory: string, pathToAcvm: string, -) { + outputFilename?: string, +): Promise { const bytecodeFilename = 'bytecode'; const witnessFilename = 'input_witness.toml'; @@ -47,55 +71,69 @@ export async function executeNativeCircuit( witnessMap = witnessMap.concat(`${key} = '${value}'\n`); }); - // In case the directory is still around from some time previously, remove it - await fs.rm(workingDirectory, { recursive: true, force: true }); - // Create the new working directory - await fs.mkdir(workingDirectory, { recursive: true }); - // Write the bytecode and input witness to the working directory - await fs.writeFile(`${workingDirectory}/${bytecodeFilename}`, bytecode); - await fs.writeFile(`${workingDirectory}/${witnessFilename}`, witnessMap); - - // Execute the ACVM using the given args - const args = [ - `execute`, - `--working-directory`, - `${workingDirectory}`, - `--bytecode`, - `${bytecodeFilename}`, - `--input-witness`, - `${witnessFilename}`, - `--print`, - ]; - const processPromise = new Promise((resolve, reject) => { - let outputWitness = Buffer.alloc(0); - let errorBuffer = Buffer.alloc(0); - const acvm = proc.spawn(pathToAcvm, args); - acvm.stdout.on('data', data => { - outputWitness = Buffer.concat([outputWitness, data]); - }); - acvm.stderr.on('data', data => { - errorBuffer = Buffer.concat([errorBuffer, data]); - }); - acvm.on('close', code => { - if (code === 0) { - resolve(outputWitness.toString('utf-8')); - } else { - reject(errorBuffer.toString('utf-8')); - } - }); - }); + try { + // Check that the directory exists + await fs.access(workingDirectory); + } catch (error) { + return { status: ACVM_RESULT.FAILURE, reason: `Working directory ${workingDirectory} does not exist` }; + } try { + // Write the bytecode and input witness to the working directory + await fs.writeFile(`${workingDirectory}/${bytecodeFilename}`, bytecode); + await fs.writeFile(`${workingDirectory}/${witnessFilename}`, witnessMap); + + // Execute the ACVM using the given args + const args = [ + `execute`, + `--working-directory`, + `${workingDirectory}`, + `--bytecode`, + `${bytecodeFilename}`, + `--input-witness`, + `${witnessFilename}`, + '--print', + '--output-witness', + 'output-witness', + ]; + + logger.debug(`Calling ACVM with ${args.join(' ')}`); + + const processPromise = new Promise((resolve, reject) => { + let outputWitness = Buffer.alloc(0); + let errorBuffer = Buffer.alloc(0); + const acvm = proc.spawn(pathToAcvm, args); + acvm.stdout.on('data', data => { + outputWitness = Buffer.concat([outputWitness, data]); + }); + acvm.stderr.on('data', data => { + errorBuffer = Buffer.concat([errorBuffer, data]); + }); + acvm.on('close', code => { + if (code === 0) { + resolve(outputWitness.toString('utf-8')); + } else { + logger.error(`From ACVM: ${errorBuffer.toString('utf-8')}`); + reject(errorBuffer.toString('utf-8')); + } + }); + }); + + const duration = new Timer(); const output = await processPromise; - return parseIntoWitnessMap(output); - } finally { - // Clean up the working directory before we leave - await fs.rm(workingDirectory, { recursive: true, force: true }); + if (outputFilename) { + const outputWitnessFileName = `${workingDirectory}/output-witness.gz`; + await fs.copyFile(outputWitnessFileName, outputFilename); + } + const witness = parseIntoWitnessMap(output); + return { status: ACVM_RESULT.SUCCESS, witness, duration: duration.ms() }; + } catch (error) { + return { status: ACVM_RESULT.FAILURE, reason: `${error}` }; } } export class NativeACVMSimulator implements SimulationProvider { - constructor(private workingDirectory: string, private pathToAcvm: string) {} + constructor(private workingDirectory: string, private pathToAcvm: string, private witnessFilename?: string) {} async simulateCircuit(input: WitnessMap, compiledCircuit: NoirCompiledCircuit): Promise { // Execute the circuit on those initial witness values @@ -103,10 +141,19 @@ export class NativeACVMSimulator implements SimulationProvider { const decodedBytecode = Buffer.from(compiledCircuit.bytecode, 'base64'); // Provide a unique working directory so we don't get clashes with parallel executions - const directory = `${this.workingDirectory}/${randomBytes(32).toString('hex')}`; + const directory = `${this.workingDirectory}/${randomBytes(8).toString('hex')}`; + + await fs.mkdir(directory, { recursive: true }); + // Execute the circuit - const _witnessMap = await executeNativeCircuit(input, decodedBytecode, directory, this.pathToAcvm); + const result = await executeNativeCircuit(input, decodedBytecode, directory, this.pathToAcvm, this.witnessFilename); + + await fs.rm(directory, { force: true, recursive: true }); + + if (result.status == ACVM_RESULT.FAILURE) { + throw new Error(`Failed to generate witness: ${result.reason}`); + } - return _witnessMap; + return result.witness; } } diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 5cdbc5d3b5a0..4f58d8e7e66e 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -755,12 +755,17 @@ __metadata: "@types/jest": ^29.5.0 "@types/memdown": ^3.0.0 "@types/node": ^18.7.23 + "@types/source-map-support": ^0.5.10 + commander: ^9.0.0 jest: ^29.5.0 jest-mock-extended: ^3.0.3 lodash.chunk: ^4.2.0 + source-map-support: ^0.5.21 ts-node: ^10.9.1 tslib: ^2.4.0 typescript: ^5.0.4 + bin: + bb-cli: ./dest/bb/index.js languageName: unknown linkType: soft