From 99a761d08a82be65b843968d71e34f97a9342529 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 13 Aug 2024 10:38:48 -0300 Subject: [PATCH 1/3] chore: Add env var to disable bb cleanup Adds a new `BB_SKIP_CLEANUP` boolean env var to stop bb native prover from cleaning up tmp dirs. Useful for retrieving files after a failed run. --- yarn-project/bb-prover/src/config.ts | 2 ++ .../src/prover/bb_private_kernel_prover.ts | 13 +++++++++---- yarn-project/bb-prover/src/prover/bb_prover.ts | 16 ++++++++++------ .../bb-prover/src/verifier/bb_verifier.ts | 4 ++-- yarn-project/foundation/src/config/env_var.ts | 1 + yarn-project/foundation/src/fs/run_in_dir.ts | 4 ++-- yarn-project/pxe/src/config/index.ts | 6 ++++++ .../pxe/src/pxe_service/create_pxe_service.ts | 1 + .../simulator/src/providers/acvm_native.ts | 2 +- 9 files changed, 34 insertions(+), 15 deletions(-) diff --git a/yarn-project/bb-prover/src/config.ts b/yarn-project/bb-prover/src/config.ts index c097693e5447..9252dbdd0389 100644 --- a/yarn-project/bb-prover/src/config.ts +++ b/yarn-project/bb-prover/src/config.ts @@ -1,6 +1,8 @@ export interface BBConfig { bbBinaryPath: string; bbWorkingDirectory: string; + /** Whether to skip tmp dir cleanup for debugging purposes */ + bbSkipCleanup?: boolean; } export interface ACVMConfig { diff --git a/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts b/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts index 80dc0e49bfc1..e451d75746d0 100644 --- a/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts +++ b/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts @@ -76,6 +76,7 @@ export class BBNativePrivateKernelProver implements PrivateKernelProver { constructor( private bbBinaryPath: string, private bbWorkingDirectory: string, + private skipCleanup: boolean, private log = createDebugLogger('aztec:bb-native-prover'), ) {} @@ -119,7 +120,7 @@ export class BBNativePrivateKernelProver implements PrivateKernelProver { const operation = async (directory: string) => { return await this._createClientIvcProof(directory, acirs, witnessStack); }; - return await runInDirectory(this.bbWorkingDirectory, operation); + return await this.runInDirectory(operation); } public getSiloedCommitments(publicInputs: PrivateCircuitPublicInputs) { @@ -189,7 +190,7 @@ export class BBNativePrivateKernelProver implements PrivateKernelProver { return await this.computeVerificationKey(directory, bytecode, 'App', appCircuitName); }; - return await runInDirectory(this.bbWorkingDirectory, operation); + return await this.runInDirectory(operation); } /** @@ -229,7 +230,7 @@ export class BBNativePrivateKernelProver implements PrivateKernelProver { await fs.writeFile(verificationKeyPath, verificationKey); return await verifyProof(this.bbBinaryPath, proofFileName, verificationKeyPath!, logFunction); }; - return await runInDirectory(this.bbWorkingDirectory, operation); + return await this.runInDirectory(operation); } /** @@ -270,7 +271,7 @@ export class BBNativePrivateKernelProver implements PrivateKernelProver { } satisfies CircuitWitnessGenerationStats); // TODO(#7410) we dont need to generate vk's for these circuits, they are in the vk tree - const { verificationKey } = await runInDirectory(this.bbWorkingDirectory, dir => + const { verificationKey } = await this.runInDirectory(dir => this.computeVerificationKey(dir, Buffer.from(compiledCircuit.bytecode, 'base64'), circuitType), ); const kernelOutput: PrivateKernelSimulateOutput = { @@ -363,4 +364,8 @@ export class BBNativePrivateKernelProver implements PrivateKernelProver { ); return proof; } + + private runInDirectory(fn: (dir: string) => Promise) { + return runInDirectory(this.bbWorkingDirectory, fn, this.skipCleanup); + } } diff --git a/yarn-project/bb-prover/src/prover/bb_prover.ts b/yarn-project/bb-prover/src/prover/bb_prover.ts index 1e1325fa4a3b..d7275812e25c 100644 --- a/yarn-project/bb-prover/src/prover/bb_prover.ts +++ b/yarn-project/bb-prover/src/prover/bb_prover.ts @@ -538,7 +538,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { return { circuitOutput: output, proof }; }; - return await runInDirectory(this.config.bbWorkingDirectory, operation); + return await this.runInDirectory(operation); } private async generateAvmProofWithBB(input: AvmCircuitInputs, workingDirectory: string): Promise { @@ -608,7 +608,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { return { proof, verificationKey }; }; - return await runInDirectory(this.config.bbWorkingDirectory, operation, cleanupDir); + return await this.runInDirectory(operation); } public async getTubeProof( @@ -640,7 +640,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { return { tubeVK, tubeProof }; }; - return await runInDirectory(this.config.bbWorkingDirectory, operation); + return await this.runInDirectory(operation); } /** @@ -699,7 +699,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { proof, }; }; - return await runInDirectory(this.config.bbWorkingDirectory, operation); + return await this.runInDirectory(operation); } /** @@ -752,7 +752,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { logger.info(`Successfully verified proof from key in ${result.durationMs} ms`); }; - await runInDirectory(this.config.bbWorkingDirectory, operation); + await this.runInDirectory(operation); } /** @@ -824,7 +824,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { true, ); }; - return await runInDirectory(this.config.bbWorkingDirectory, operation); + return await this.runInDirectory(operation); } /** @@ -958,4 +958,8 @@ export class BBNativeRollupProver implements ServerCircuitProver { return proof; } + + private runInDirectory(fn: (dir: string) => Promise) { + return runInDirectory(this.config.bbWorkingDirectory, fn, this.config.bbSkipCleanup); + } } diff --git a/yarn-project/bb-prover/src/verifier/bb_verifier.ts b/yarn-project/bb-prover/src/verifier/bb_verifier.ts index b09c2d981c52..52ddf8129abb 100644 --- a/yarn-project/bb-prover/src/verifier/bb_verifier.ts +++ b/yarn-project/bb-prover/src/verifier/bb_verifier.ts @@ -116,7 +116,7 @@ export class BBCircuitVerifier implements ClientProtocolCircuitVerifier { proofType: 'ultra-honk', } satisfies CircuitVerificationStats); }; - await runInDirectory(this.config.bbWorkingDirectory, operation); + await runInDirectory(this.config.bbWorkingDirectory, operation, this.config.bbSkipCleanup); } public async generateSolidityContract(circuit: ProtocolArtifact, contractName: string) { @@ -168,7 +168,7 @@ export class BBCircuitVerifier implements ClientProtocolCircuitVerifier { proofType: 'client-ivc', } satisfies CircuitVerificationStats); }; - await runInDirectory(this.config.bbWorkingDirectory, operation); + await runInDirectory(this.config.bbWorkingDirectory, operation, this.config.bbSkipCleanup); return true; } catch (err) { this.logger.warn(`Failed to verify ClientIVC proof for tx ${Tx.getHash(tx)}: ${String(err)}`); diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 4e15a54727bb..233e89328acf 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -95,4 +95,5 @@ export type EnvVar = | 'PXE_DATA_DIRECTORY' | 'BB_BINARY_PATH' | 'BB_WORKING_DIRECTORY' + | 'BB_SKIP_CLEANUP' | 'PXE_PROVER_ENABLED'; diff --git a/yarn-project/foundation/src/fs/run_in_dir.ts b/yarn-project/foundation/src/fs/run_in_dir.ts index 7b39b27aca8d..eb4794c8c647 100644 --- a/yarn-project/foundation/src/fs/run_in_dir.ts +++ b/yarn-project/foundation/src/fs/run_in_dir.ts @@ -6,7 +6,7 @@ import * as path from 'path'; export async function runInDirectory( workingDirBase: string, fn: (dir: string) => Promise, - cleanup: boolean = true, + skipCleanup: boolean | undefined, ): Promise { // Create random directory to be used for temp files const workingDirectory = await fs.mkdtemp(path.join(workingDirBase, 'tmp-')); @@ -16,7 +16,7 @@ export async function runInDirectory( try { return await fn(workingDirectory); } finally { - if (cleanup) { + if (!skipCleanup) { await fs.rm(workingDirectory, { recursive: true, force: true }); } } diff --git a/yarn-project/pxe/src/config/index.ts b/yarn-project/pxe/src/config/index.ts index dbc82c4fa702..0873a8454143 100644 --- a/yarn-project/pxe/src/config/index.ts +++ b/yarn-project/pxe/src/config/index.ts @@ -17,6 +17,7 @@ import { fileURLToPath } from 'url'; export interface BBProverConfig { bbWorkingDirectory?: string; bbBinaryPath?: string; + bbSkipCleanup?: boolean; } /** @@ -72,6 +73,11 @@ export const pxeConfigMappings: ConfigMappingsType = { env: 'BB_WORKING_DIRECTORY', description: 'Working directory for the BB binary', }, + bbSkipCleanup: { + env: 'BB_SKIP_CLEANUP', + description: 'True to skip cleanup of temporary files for debugging purposes', + ...booleanConfigHelper(), + }, proverEnabled: { env: 'PXE_PROVER_ENABLED', description: 'Enable real proofs', diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts index 23234b509c20..62c15081accc 100644 --- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts @@ -57,6 +57,7 @@ export async function createPXEService( : new BBNativePrivateKernelProver( config.bbBinaryPath!, config.bbWorkingDirectory!, + !!config.bbSkipCleanup, createDebugLogger('aztec:pxe:bb-native-prover' + (logSuffix ? `:${logSuffix}` : '')), ); } diff --git a/yarn-project/simulator/src/providers/acvm_native.ts b/yarn-project/simulator/src/providers/acvm_native.ts index be04ca57a63c..3bf1bdf01572 100644 --- a/yarn-project/simulator/src/providers/acvm_native.ts +++ b/yarn-project/simulator/src/providers/acvm_native.ts @@ -156,6 +156,6 @@ export class NativeACVMSimulator implements SimulationProvider { return result.witness; }; - return await runInDirectory(this.workingDirectory, operation); + return await runInDirectory(this.workingDirectory, operation, false); } } From fd7c9898ff28b40da9e25fde496c9f2ca2e3ef87 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 13 Aug 2024 10:43:53 -0300 Subject: [PATCH 2/3] Honor AVM_PROVING_PRESERVE_WORKING_DIR --- yarn-project/bb-prover/src/prover/bb_prover.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yarn-project/bb-prover/src/prover/bb_prover.ts b/yarn-project/bb-prover/src/prover/bb_prover.ts index d7275812e25c..aa52c7d9f580 100644 --- a/yarn-project/bb-prover/src/prover/bb_prover.ts +++ b/yarn-project/bb-prover/src/prover/bb_prover.ts @@ -571,9 +571,10 @@ export class BBNativeRollupProver implements ServerCircuitProver { } private async createAvmProof(input: AvmCircuitInputs): Promise { - const cleanupDir: boolean = !process.env.AVM_PROVING_PRESERVE_WORKING_DIR; + const skipCleanup = + ['1', 'true'].includes(process.env.AVM_PROVING_PRESERVE_WORKING_DIR ?? '') || !!this.config.bbSkipCleanup; const operation = async (bbWorkingDirectory: string): Promise => { - if (!cleanupDir) { + if (skipCleanup) { logger.info(`Preserving working directory ${bbWorkingDirectory}`); } const provingResult = await this.generateAvmProofWithBB(input, bbWorkingDirectory); @@ -608,7 +609,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { return { proof, verificationKey }; }; - return await this.runInDirectory(operation); + return await runInDirectory(this.config.bbWorkingDirectory, operation, skipCleanup); } public async getTubeProof( From c8ce18af8fb0b9e98b15b62c1be684b85fe5e56c Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 13 Aug 2024 14:51:58 -0300 Subject: [PATCH 3/3] Remove AVM_PROVING_PRESERVE_WORKING_DIR --- yarn-project/bb-prover/src/prover/bb_prover.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/yarn-project/bb-prover/src/prover/bb_prover.ts b/yarn-project/bb-prover/src/prover/bb_prover.ts index aa52c7d9f580..e432628c054f 100644 --- a/yarn-project/bb-prover/src/prover/bb_prover.ts +++ b/yarn-project/bb-prover/src/prover/bb_prover.ts @@ -571,12 +571,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { } private async createAvmProof(input: AvmCircuitInputs): Promise { - const skipCleanup = - ['1', 'true'].includes(process.env.AVM_PROVING_PRESERVE_WORKING_DIR ?? '') || !!this.config.bbSkipCleanup; const operation = async (bbWorkingDirectory: string): Promise => { - if (skipCleanup) { - logger.info(`Preserving working directory ${bbWorkingDirectory}`); - } const provingResult = await this.generateAvmProofWithBB(input, bbWorkingDirectory); const rawProof = await fs.readFile(provingResult.proofPath!); @@ -609,7 +604,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { return { proof, verificationKey }; }; - return await runInDirectory(this.config.bbWorkingDirectory, operation, skipCleanup); + return await this.runInDirectory(operation); } public async getTubeProof(