From 121c23c1a2ade7c1403bcef7991d1272d106ef1f Mon Sep 17 00:00:00 2001 From: thunkar Date: Fri, 23 May 2025 13:36:21 +0000 Subject: [PATCH 1/8] gj/app_benches diff --- bootstrap.sh | 3 +- yarn-project/bb-prover/package.json | 2 + yarn-project/bb-prover/src/bb/execute.ts | 17 +- yarn-project/end-to-end/bootstrap.sh | 8 +- .../client_flows/account_deployments.test.ts | 20 +- .../src/bench/client_flows/amm.test.ts | 10 +- .../src/bench/client_flows/bridging.test.ts | 18 +- .../client_flows/client_flows_benchmark.ts | 38 ++- .../src/bench/client_flows/data_extractor.ts | 150 +--------- .../bench/client_flows/deployments.test.ts | 11 +- .../src/bench/client_flows/transfers.test.ts | 65 +++-- .../src/bench/client_flows/utils.ts | 267 ++++++++++++++++++ yarn-project/end-to-end/src/bench/utils.ts | 2 +- yarn-project/end-to-end/src/e2e_amm.test.ts | 3 - .../transfer_private.test.ts | 2 - yarn-project/end-to-end/src/e2e_nft.test.ts | 3 - .../end-to-end/src/e2e_p2p/add_rollup.test.ts | 6 +- yarn-project/end-to-end/src/e2e_p2p/shared.ts | 4 +- yarn-project/end-to-end/src/fixtures/utils.ts | 2 +- .../shared/capture_private_execution_steps.ts | 68 ----- .../src/entrypoints/client/bundle/utils.ts | 36 ++- .../pxe/src/entrypoints/client/lazy/utils.ts | 33 ++- .../{client => }/pxe_creation_options.ts | 5 +- .../pxe/src/entrypoints/server/utils.ts | 44 +-- yarn-project/yarn.lock | 18 ++ 25 files changed, 497 insertions(+), 338 deletions(-) create mode 100644 yarn-project/end-to-end/src/bench/client_flows/utils.ts delete mode 100644 yarn-project/end-to-end/src/shared/capture_private_execution_steps.ts rename yarn-project/pxe/src/entrypoints/{client => }/pxe_creation_options.ts (51%) diff --git a/bootstrap.sh b/bootstrap.sh index 408786682ffc..5840799aac1e 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -253,8 +253,7 @@ function build_bench { return fi parallel --line-buffer --tag --halt now,fail=1 'denoise "{}/bootstrap.sh build_bench"' ::: \ - barretenberg/cpp \ - yarn-project/end-to-end + barretenberg/cpp } export -f build_bench diff --git a/yarn-project/bb-prover/package.json b/yarn-project/bb-prover/package.json index 3fa6b015818c..a591fb8caf0f 100644 --- a/yarn-project/bb-prover/package.json +++ b/yarn-project/bb-prover/package.json @@ -78,6 +78,7 @@ "@aztec/world-state": "workspace:^", "commander": "^12.1.0", "pako": "^2.1.0", + "pidusage": "^4.0.1", "source-map-support": "^0.5.21", "tslib": "^2.4.0" }, @@ -90,6 +91,7 @@ "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", "@types/node": "^22.15.17", + "@types/pidusage": "^2.0.5", "@types/source-map-support": "^0.5.10", "jest": "^29.5.0", "jest-mock-extended": "^3.0.3", diff --git a/yarn-project/bb-prover/src/bb/execute.ts b/yarn-project/bb-prover/src/bb/execute.ts index 45390431102c..52586062fbca 100644 --- a/yarn-project/bb-prover/src/bb/execute.ts +++ b/yarn-project/bb-prover/src/bb/execute.ts @@ -6,6 +6,7 @@ import type { AvmCircuitInputs, AvmCircuitPublicInputs } from '@aztec/stdlib/avm import * as proc from 'child_process'; import { promises as fs } from 'fs'; import { basename, dirname, join } from 'path'; +import pidusage from 'pidusage'; import type { UltraHonkFlavor } from '../honk.js'; @@ -97,11 +98,23 @@ export function executeBB( bb.stdout.on('data', data => { const message = data.toString('utf-8').replace(/\n$/, ''); - logger(message); + pidusage(bb.pid!, (err, stats) => { + if (err) { + logger(message); + } else { + logger(`${message} (mem: ${(stats.memory / 1024 / 1024).toFixed(2)}MiB)`); + } + }); }); bb.stderr.on('data', data => { const message = data.toString('utf-8').replace(/\n$/, ''); - logger(message); + pidusage(bb.pid!, (err, stats) => { + if (err) { + logger(message); + } else { + logger(`${message} (mem: ${(stats.memory / 1024 / 1024).toFixed(2)}MiB)`); + } + }); }); bb.on('close', (exitCode: number, signal?: string) => { if (timeoutId) { diff --git a/yarn-project/end-to-end/bootstrap.sh b/yarn-project/end-to-end/bootstrap.sh index b3ede6f92936..2f4a480998d2 100755 --- a/yarn-project/end-to-end/bootstrap.sh +++ b/yarn-project/end-to-end/bootstrap.sh @@ -59,10 +59,9 @@ function test { function bench_cmds { echo "$hash:ISOLATE=1:NAME=bench_build_block BENCH_OUTPUT=bench-out/build-block.bench.json yarn-project/end-to-end/scripts/run_test.sh simple bench_build_block" - for dir in $bench_fixtures_dir/*; do - for runtime in native wasm; do - echo "$hash:CPUS=8 barretenberg/cpp/scripts/ci_benchmark_ivc_flows.sh $runtime ../../yarn-project/end-to-end/$dir" - done + + for client_flow in client_flows/bridging client_flows/deployments client_flows/amm client_flows/account_deployments client_flows/transfers; do + echo "$hash:ISOLATE=1:CPUS=8:NAME=$client_flow REAL_PROOFS=1 BENCHMARK_CONFIG=key_flows LOG_LEVEL=error BENCH_OUTPUT=bench-out/ yarn-project/end-to-end/scripts/run_test.sh simple $client_flow" done } @@ -82,6 +81,7 @@ function build_bench { exit 1 fi parallel --tag --line-buffer --halt now,fail=1 'docker_isolate "scripts/run_test.sh simple {}"' ::: \ + client_flows/account_deployments \ client_flows/deployments \ client_flows/bridging \ client_flows/transfers \ diff --git a/yarn-project/end-to-end/src/bench/client_flows/account_deployments.test.ts b/yarn-project/end-to-end/src/bench/client_flows/account_deployments.test.ts index c48c4843568d..650e7ab9365c 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/account_deployments.test.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/account_deployments.test.ts @@ -1,11 +1,11 @@ import { EcdsaRAccountContractArtifact } from '@aztec/accounts/ecdsa'; -import { AccountWallet, type DeployOptions, Fr, registerContractClass } from '@aztec/aztec.js'; +import { AccountWallet, type DeployOptions, Fr, type PXE, registerContractClass } from '@aztec/aztec.js'; import type { SponsoredFPCContract } from '@aztec/noir-contracts.js/SponsoredFPC'; import { jest } from '@jest/globals'; -import { capturePrivateExecutionStepsIfEnvSet } from '../../shared/capture_private_execution_steps.js'; import { type AccountType, type BenchmarkingFeePaymentMethod, ClientFlowsBenchmark } from './client_flows_benchmark.js'; +import { captureProfile } from './utils.js'; jest.setTimeout(300_000); @@ -17,11 +17,13 @@ describe('Deployment benchmark', () => { let sponsoredFPC: SponsoredFPCContract; // Benchmarking configuration const config = t.config.accountDeployments; + // Benchmarking user's PXE + let userPXE: PXE; beforeAll(async () => { await t.applyBaseSnapshots(); await t.applyDeploySponsoredFPCSnapshot(); - ({ adminWallet, sponsoredFPC } = await t.setup()); + ({ adminWallet, sponsoredFPC, userPXE } = await t.setup()); // Ensure the ECDSAR1 contract is already registered, to avoid benchmarking an extra call to the ContractClassRegisterer // The typical interaction would be for a user to deploy an account contract that is already registered in the // network. @@ -41,7 +43,7 @@ describe('Deployment benchmark', () => { return describe(`Deployment benchmark for ${accountType}`, () => { function deploymentTest(benchmarkingPaymentMethod: BenchmarkingFeePaymentMethod) { return it(`Deploys a ${accountType} account contract, pays using ${benchmarkingPaymentMethod}`, async () => { - const benchysAccountManager = await t.createBenchmarkingAccountManager(accountType); + const benchysAccountManager = await t.createBenchmarkingAccountManager(userPXE, accountType); const benchysWallet = await benchysAccountManager.getWallet(); if (benchmarkingPaymentMethod === 'sponsored_fpc') { @@ -66,7 +68,7 @@ describe('Deployment benchmark', () => { contractAddressSalt: new Fr(benchysAccountManager.salt), }; - await capturePrivateExecutionStepsIfEnvSet( + await captureProfile( `deploy_${accountType}+${benchmarkingPaymentMethod}`, deploymentInteraction, options, @@ -81,9 +83,11 @@ describe('Deployment benchmark', () => { 1, // Kernel tail ); - // Ensure we paid a fee - const tx = await deploymentInteraction.send(options).wait(); - expect(tx.transactionFee!).toBeGreaterThan(0n); + if (process.env.SANITY_CHECKS) { + // Ensure we paid a fee + const tx = await deploymentInteraction.send(options).wait(); + expect(tx.transactionFee!).toBeGreaterThan(0n); + } }); } diff --git a/yarn-project/end-to-end/src/bench/client_flows/amm.test.ts b/yarn-project/end-to-end/src/bench/client_flows/amm.test.ts index baba8b954c39..dbc2c27991c4 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/amm.test.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/amm.test.ts @@ -8,8 +8,8 @@ import { TokenContract } from '@aztec/noir-contracts.js/Token'; import { jest } from '@jest/globals'; import { mintNotes } from '../../fixtures/token_utils.js'; -import { capturePrivateExecutionStepsIfEnvSet } from '../../shared/capture_private_execution_steps.js'; import { type AccountType, type BenchmarkingFeePaymentMethod, ClientFlowsBenchmark } from './client_flows_benchmark.js'; +import { captureProfile } from './utils.js'; jest.setTimeout(900_000); @@ -132,7 +132,7 @@ describe('AMM benchmark', () => { .methods.add_liquidity(amountToSend, amountToSend, amountToSend, amountToSend, nonceForAuthwits) .with({ authWitnesses: [token0Authwit, token1Authwit] }); - await capturePrivateExecutionStepsIfEnvSet( + await captureProfile( `${accountType}+amm_add_liquidity_1_recursions+${benchmarkingPaymentMethod}`, addLiquidityInteraction, options, @@ -151,8 +151,10 @@ describe('AMM benchmark', () => { 1, // Kernel tail ); - const tx = await addLiquidityInteraction.send().wait(); - expect(tx.transactionFee!).toBeGreaterThan(0n); + if (process.env.SANITY_CHECKS) { + const tx = await addLiquidityInteraction.send().wait(); + expect(tx.transactionFee!).toBeGreaterThan(0n); + } }); }); } diff --git a/yarn-project/end-to-end/src/bench/client_flows/bridging.test.ts b/yarn-project/end-to-end/src/bench/client_flows/bridging.test.ts index 851ba75091f9..a348f0b622b4 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/bridging.test.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/bridging.test.ts @@ -6,9 +6,9 @@ import { TokenContract } from '@aztec/noir-contracts.js/Token'; import { jest } from '@jest/globals'; -import { capturePrivateExecutionStepsIfEnvSet } from '../../shared/capture_private_execution_steps.js'; import type { CrossChainTestHarness } from '../../shared/cross_chain_test_harness.js'; import { type AccountType, type BenchmarkingFeePaymentMethod, ClientFlowsBenchmark } from './client_flows_benchmark.js'; +import { captureProfile } from './utils.js'; jest.setTimeout(300_000); @@ -91,7 +91,7 @@ describe('Bridging benchmark', () => { messageLeafIndex, ); - await capturePrivateExecutionStepsIfEnvSet( + await captureProfile( `${accountType}+token_bridge_claim_private+${benchmarkingPaymentMethod}`, claimInteraction, options, @@ -104,14 +104,16 @@ describe('Bridging benchmark', () => { 1, // Kernel tail ); - // Ensure we paid a fee - const tx = await claimInteraction.send(options).wait(); - expect(tx.transactionFee!).toBeGreaterThan(0n); + if (process.env.SANITY_CHECKS) { + // Ensure we paid a fee + const tx = await claimInteraction.send(options).wait(); + expect(tx.transactionFee!).toBeGreaterThan(0n); - // 4. Check the balance + // 4. Check the balance - const balance = await crossChainTestHarness.getL2PrivateBalanceOf(benchysWallet.getAddress()); - expect(balance).toBe(bridgeAmount); + const balance = await crossChainTestHarness.getL2PrivateBalanceOf(benchysWallet.getAddress()); + expect(balance).toBe(bridgeAmount); + } }); } diff --git a/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts b/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts index b6143aed1692..4563713837b3 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts @@ -2,6 +2,7 @@ import { EcdsaRAccountContractArtifact, getEcdsaRAccount } from '@aztec/accounts import { SchnorrAccountContractArtifact, getSchnorrAccount, getSchnorrWallet } from '@aztec/accounts/schnorr'; import { type AccountWallet, + AccountWalletWithSecretKey, AztecAddress, type AztecNode, FeeJuicePaymentMethod, @@ -48,6 +49,7 @@ import { type GasBridgingTestHarness, } from '../../shared/gas_portal_test_harness.js'; import { type ClientFlowsConfig, FULL_FLOWS_CONFIG, KEY_FLOWS_CONFIG } from './config.js'; +import { ProxyLogger } from './utils.js'; const { E2E_DATA_PATH: dataPath, BENCHMARK_CONFIG } = process.env; @@ -91,6 +93,8 @@ export class ClientFlowsBenchmark { // PXE used by the benchmarking user. It can be set up with client-side proving enabled public userPXE!: PXE; + public realProofs = ['true', '1'].includes(process.env.REAL_PROOFS ?? ''); + public paymentMethods: Record = { // eslint-disable-next-line camelcase @@ -121,6 +125,8 @@ export class ClientFlowsBenchmark { public config: ClientFlowsConfig; + private proxyLogger: ProxyLogger; + constructor(testName?: string, setupOptions: Partial = {}) { this.logger = createLogger(`bench:client_flows${testName ? `:${testName}` : ''}`); this.snapshotManager = createSnapshotManager( @@ -130,6 +136,8 @@ export class ClientFlowsBenchmark { { ...setupOptions }, ); this.config = BENCHMARK_CONFIG === 'key_flows' ? KEY_FLOWS_CONFIG : FULL_FLOWS_CONFIG; + ProxyLogger.create(); + this.proxyLogger = ProxyLogger.getInstance(); } async setup() { @@ -163,7 +171,7 @@ export class ClientFlowsBenchmark { expect(balanceAfter).toEqual(balanceBefore + amount); } - async createBenchmarkingAccountManager(type: 'ecdsar1' | 'schnorr') { + async createBenchmarkingAccountManager(pxe: PXE, type: 'ecdsar1' | 'schnorr') { const benchysSecretKey = Fr.random(); const salt = Fr.random(); @@ -171,10 +179,10 @@ export class ClientFlowsBenchmark { let benchysAccountManager; if (type === 'schnorr') { benchysPrivateSigningKey = deriveSigningKey(benchysSecretKey); - benchysAccountManager = await getSchnorrAccount(this.userPXE, benchysSecretKey, benchysPrivateSigningKey, salt); + benchysAccountManager = await getSchnorrAccount(pxe, benchysSecretKey, benchysPrivateSigningKey, salt); } else if (type === 'ecdsar1') { benchysPrivateSigningKey = randomBytes(32); - benchysAccountManager = await getEcdsaRAccount(this.userPXE, benchysSecretKey, benchysPrivateSigningKey, salt); + benchysAccountManager = await getEcdsaRAccount(pxe, benchysSecretKey, benchysPrivateSigningKey, salt); } else { throw new Error(`Unknown account type: ${type}`); } @@ -212,11 +220,15 @@ export class ClientFlowsBenchmark { const l1Contracts = await aztecNode.getL1ContractAddresses(); const userPXEConfigWithContracts = { ...userPXEConfig, - proverEnabled: ['true', '1'].includes(process.env.REAL_PROOFS ?? ''), + proverEnabled: this.realProofs, l1Contracts, } as PXEServiceConfig; - this.userPXE = await createPXEService(this.aztecNode, userPXEConfigWithContracts, 'pxe-user'); + this.userPXE = await createPXEService(this.aztecNode, userPXEConfigWithContracts, { + loggers: { + prover: this.proxyLogger.createLogger('pxe:bb:wasm:bundle:proxied'), + }, + }); }, ); } @@ -337,7 +349,7 @@ export class ClientFlowsBenchmark { } public async createAndFundBenchmarkingWallet(accountType: AccountType) { - const benchysAccountManager = await this.createBenchmarkingAccountManager(accountType); + const benchysAccountManager = await this.createBenchmarkingAccountManager(this.pxe, accountType); const benchysWallet = await benchysAccountManager.getWallet(); const benchysAddress = benchysAccountManager.getAddress(); const claim = await this.feeJuiceBridgeTestHarness.prepareTokensOnL1( @@ -346,13 +358,19 @@ export class ClientFlowsBenchmark { ); const paymentMethod = new FeeJuicePaymentMethodWithClaim(benchysWallet, claim); await benchysAccountManager.deploy({ fee: { paymentMethod } }).wait(); - // Register benchy on admin's PXE so we can check its balances - await this.pxe.registerContract({ + // Register benchy on the user's PXE, where we're going to be interacting from + await this.userPXE.registerContract({ instance: benchysAccountManager.getInstance(), artifact: accountType === 'ecdsar1' ? EcdsaRAccountContractArtifact : SchnorrAccountContractArtifact, }); - await this.pxe.registerAccount(benchysWallet.getSecretKey(), benchysWallet.getCompleteAddress().partialAddress); - return benchysWallet; + await this.userPXE.registerAccount(benchysWallet.getSecretKey(), benchysWallet.getCompleteAddress().partialAddress); + const entrypoint = await benchysAccountManager.getAccount(); + return new AccountWalletWithSecretKey( + this.userPXE, + entrypoint, + benchysWallet.getSecretKey(), + benchysAccountManager.salt, + ); } public async applyDeployAmmSnapshot() { diff --git a/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts b/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts index 197bd87f9d3f..3531ad9e1731 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts @@ -5,94 +5,16 @@ import { createLogger, logger } from '@aztec/foundation/log'; import { Timer } from '@aztec/foundation/timer'; import { WASMSimulator } from '@aztec/simulator/client'; import type { PrivateExecutionStep } from '@aztec/stdlib/kernel'; +import type { ProvingTimings, SimulationTimings } from '@aztec/stdlib/tx'; import { Decoder } from 'msgpackr'; -import assert from 'node:assert'; import { readFile, readdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; -const logLevel = ['silent', 'fatal', 'error', 'warn', 'info', 'verbose', 'debug', 'trace'] as const; -type LogLevel = (typeof logLevel)[number]; - -type Log = { - type: LogLevel; - timestamp: number; - prefix: string; - message: string; - - data: any; -}; - -const GATE_TYPES = [ - 'ecc_op', - 'busread', - 'lookup', - 'pub_inputs', - 'arithmetic', - 'delta_range', - 'elliptic', - 'aux', - 'poseidon2_external', - 'poseidon2_internal', - 'overflow', -] as const; - -type GateType = (typeof GATE_TYPES)[number]; - -type StructuredTrace = { - [k in GateType]: number; -}; - -export class ProxyLogger { - private static instance: ProxyLogger; - private logs: Log[] = []; - - private constructor() {} - - static create() { - ProxyLogger.instance = new ProxyLogger(); - } - - static getInstance() { - return ProxyLogger.instance; - } - - createLogger(prefix: string): Logger { - return new Proxy(createLogger(prefix), { - get: (target: Logger, prop: keyof Logger) => { - if (logLevel.includes(prop as (typeof logLevel)[number])) { - return function (this: Logger, ...data: Parameters) { - const loggingFn = prop as LogLevel; - const args = [loggingFn, prefix, ...data] as Parameters; - ProxyLogger.getInstance().handleLog(...args); - target[loggingFn].call(this, ...[data[0], data[1]]); - }; - } else { - return target[prop]; - } - }, - }); - } - - private handleLog(type: (typeof logLevel)[number], prefix: string, message: string, data: any) { - this.logs.unshift({ type, prefix, message, data, timestamp: Date.now() }); - } - - public flushLogs() { - this.logs = []; - } - - public getLogs() { - return this.logs; - } -} +import { type Log, type ProverType, ProxyLogger, generateBenchmark } from './utils.js'; type NativeProverConfig = { bbBinaryPath?: string; bbWorkingDirectory?: string }; -type ProverType = 'wasm' | 'native'; - -type Step = { fnName: string; gateCount: number; accGateCount?: number }; - async function createProver(config: NativeProverConfig = {}, log: Logger) { const simulationProvider = new WASMSimulator(); if (!config.bbBinaryPath || !config.bbWorkingDirectory) { @@ -106,33 +28,6 @@ async function createProver(config: NativeProverConfig = {}, log: Logger) { } } -function getMinimumTrace(logs: Log[]): StructuredTrace { - const minimumMessage = 'Trace details:'; - const minimumMessageIndex = logs.findIndex(log => log.message.includes(minimumMessage)); - const candidateLogs = logs.slice(minimumMessageIndex - GATE_TYPES.length, minimumMessageIndex); - - const traceLogs = candidateLogs - .filter(log => GATE_TYPES.some(type => log.message.includes(type))) - .map(log => log.message.split(/\t|\n/)) - .flat() - .map(log => log.replace(/\(mem: .*\)/, '').trim()) - .filter(Boolean); - - const traceSizes = traceLogs.map(log => { - const [gateType, gateSizeStr] = log - .replace(/\n.*\)$/, '') - .replace(/bb - /, '') - .split(':') - .map(s => s.trim()); - const gateSize = parseInt(gateSizeStr); - assert(GATE_TYPES.includes(gateType as GateType), `Gate type ${gateType} is not recognized`); - return { [gateType]: gateSize }; - }); - - assert(traceSizes.length === GATE_TYPES.length, 'Decoded trace sizes do not match expected amount of gate types'); - return traceSizes.reduce((acc, curr) => ({ ...acc, ...curr }), {}) as StructuredTrace; -} - async function main() { ProxyLogger.create(); const proxyLogger = ProxyLogger.getInstance(); @@ -160,15 +55,15 @@ async function main() { }); const profileFile = await readFile(join(ivcFolder, flow, 'profile.json')); const profile = JSON.parse(profileFile.toString()) as { - syncTime: number; + timings: ProvingTimings | SimulationTimings; steps: { - fnName: string; + functionName: string; gateCount: number; timings: { witgen: number; gateCount: number }; }[]; }; const privateExecutionSteps: PrivateExecutionStep[] = profile.steps.map((step, i) => ({ - functionName: step.fnName, + functionName: step.functionName, gateCount: step.gateCount, bytecode: stepsFromFile[i].bytecode, // TODO(AD) do we still want to take this from witness.json? @@ -179,7 +74,6 @@ async function main() { gateCount: step.timings.witgen, }, })); - let stats: { duration: number; eventName: string; proofSize: number } | undefined; let error: any; let currentLogs: Log[] = []; @@ -195,38 +89,10 @@ async function main() { // Extract logs from this run from the proxy and write them to disk unconditionally currentLogs = proxyLogger.getLogs(); await writeFile(join(ivcFolder, flow, 'logs.json'), JSON.stringify(currentLogs, null, 2)); - - if (!error) { - stats = currentLogs[0].data as { duration: number; eventName: string; proofSize: number }; + if (!(profile.timings as ProvingTimings).proving) { + (profile.timings as ProvingTimings).proving = provingTime; } - - const minimumTrace = getMinimumTrace(currentLogs); - - const steps = profile.steps.reduce((acc, step, i) => { - const previousAccGateCount = i === 0 ? 0 : acc[i - 1].accGateCount!; - return [ - ...acc, - { - fnName: step.fnName, - gateCount: step.gateCount, - accGateCount: previousAccGateCount + step.gateCount, - timings: { - witgen: step.timings.witgen, - }, - }, - ]; - }, []); - const totalGateCount = steps[steps.length - 1].accGateCount; - const benchmark = { - syncTime: profile.syncTime, - provingTime, - proverType, - minimumTrace: minimumTrace, - totalGateCount, - stats, - steps, - error, - }; + const benchmark = generateBenchmark(flow, currentLogs, profile.timings, privateExecutionSteps, proverType, error); await writeFile(join(ivcFolder, flow, 'benchmark.json'), JSON.stringify(benchmark, null, 2)); proxyLogger.flushLogs(); } diff --git a/yarn-project/end-to-end/src/bench/client_flows/deployments.test.ts b/yarn-project/end-to-end/src/bench/client_flows/deployments.test.ts index 2752b3882009..27f91f7d72ef 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/deployments.test.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/deployments.test.ts @@ -5,8 +5,8 @@ import { getContractClassFromArtifact } from '@aztec/stdlib/contract'; import { jest } from '@jest/globals'; -import { capturePrivateExecutionStepsIfEnvSet } from '../../shared/capture_private_execution_steps.js'; import { type AccountType, type BenchmarkingFeePaymentMethod, ClientFlowsBenchmark } from './client_flows_benchmark.js'; +import { captureProfile } from './utils.js'; jest.setTimeout(1_600_000); @@ -61,7 +61,7 @@ describe('Deployment benchmark', () => { const deploymentInteraction = EasyPrivateVotingContract.deploy(benchysWallet, benchysWallet.getAddress()); - await capturePrivateExecutionStepsIfEnvSet( + await captureProfile( `${accountType}+deploy_tokenContract_${ isClassRegistered ? 'no_registration' : 'with_registration' }+${benchmarkingPaymentMethod}`, @@ -77,8 +77,11 @@ describe('Deployment benchmark', () => { 1, // Kernel tail ); - const tx = await deploymentInteraction.send(options).wait(); - expect(tx.transactionFee!).toBeGreaterThan(0n); + if (process.env.SANITY_CHECKS) { + // Ensure we paid a fee + const tx = await deploymentInteraction.send(options).wait(); + expect(tx.transactionFee!).toBeGreaterThan(0n); + } }); }); } diff --git a/yarn-project/end-to-end/src/bench/client_flows/transfers.test.ts b/yarn-project/end-to-end/src/bench/client_flows/transfers.test.ts index a49ba677bcbc..566495cd60af 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/transfers.test.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/transfers.test.ts @@ -7,8 +7,8 @@ import { TokenContract } from '@aztec/noir-contracts.js/Token'; import { jest } from '@jest/globals'; import { mintNotes } from '../../fixtures/token_utils.js'; -import { capturePrivateExecutionStepsIfEnvSet } from '../../shared/capture_private_execution_steps.js'; import { type AccountType, type BenchmarkingFeePaymentMethod, ClientFlowsBenchmark } from './client_flows_benchmark.js'; +import { captureProfile } from './utils.js'; jest.setTimeout(1_600_000); @@ -26,10 +26,10 @@ describe('Transfer benchmark', () => { let bananaCoin: TokenContract; // CandyBarCoin Token contract, which we want to transfer let candyBarCoin: TokenContract; - // Aztec node to answer queries - let node: AztecNode; // Sponsored FPC contract let sponsoredFPC: SponsoredFPCContract; + // Aztec node + let node: AztecNode; // Benchmarking configuration const config = t.config.transfers; @@ -40,7 +40,7 @@ describe('Transfer benchmark', () => { await t.applyDeployCandyBarTokenSnapshot(); await t.applyDeploySponsoredFPCSnapshot(); - ({ adminWallet, bananaFPC, bananaCoin, candyBarCoin, aztecNode: node, sponsoredFPC } = await t.setup()); + ({ adminWallet, bananaFPC, bananaCoin, candyBarCoin, sponsoredFPC, aztecNode: node } = await t.setup()); }); afterAll(async () => { @@ -122,7 +122,7 @@ describe('Transfer benchmark', () => { const transferInteraction = asset.methods.transfer(adminWallet.getAddress(), amountToSend); - await capturePrivateExecutionStepsIfEnvSet( + await captureProfile( `${accountType}+transfer_${recursions}_recursions+${benchmarkingPaymentMethod}`, transferInteraction, options, @@ -135,34 +135,37 @@ describe('Transfer benchmark', () => { 1, // Kernel tail ); - const tx = await transferInteraction.send(options).wait(); - expect(tx.transactionFee!).toBeGreaterThan(0n); - - // Sanity checks - - const txEffects = await node.getTxEffect(tx.txHash); - - /* - * We should have created the following nullifiers: - * - One per created note - * - One for the transaction - * - One for the fee note if we're using private fpc - */ - expect(txEffects!.data.nullifiers.length).toBe( - notesToCreate + 1 + (benchmarkingPaymentMethod === 'private_fpc' ? 1 : 0), - ); - /** We should have created 4 new notes, - * - One for the recipient - * - One for the sender (with the change) - * - One for the fee if we're using private fpc - * - One for the fee refund if we're using private fpc - */ - expect(txEffects!.data.noteHashes.length).toBe(2 + (benchmarkingPaymentMethod === 'private_fpc' ? 2 : 0)); - expectedChange = totalAmount - BigInt(amountToSend); - const senderBalance = await asset.methods.balance_of_private(benchysWallet.getAddress()).simulate(); - expect(senderBalance).toEqual(expectedChange); + if (process.env.SANITY_CHECKS) { + // Ensure we paid a fee + const tx = await transferInteraction.send(options).wait(); + expect(tx.transactionFee!).toBeGreaterThan(0n); + + // Sanity checks + + const txEffects = await node.getTxEffect(tx.txHash); + + /* + * We should have created the following nullifiers: + * - One per created note + * - One for the transaction + * - One for the fee note if we're using private fpc + */ + expect(txEffects!.data.nullifiers.length).toBe( + notesToCreate + 1 + (benchmarkingPaymentMethod === 'private_fpc' ? 1 : 0), + ); + /** We should have created 4 new notes, + * - One for the recipient + * - One for the sender (with the change) + * - One for the fee if we're using private fpc + * - One for the fee refund if we're using private fpc + */ + expect(txEffects!.data.noteHashes.length).toBe(2 + (benchmarkingPaymentMethod === 'private_fpc' ? 2 : 0)); + + const senderBalance = await asset.methods.balance_of_private(benchysWallet.getAddress()).simulate(); + expect(senderBalance).toEqual(expectedChange); + } }); }); } diff --git a/yarn-project/end-to-end/src/bench/client_flows/utils.ts b/yarn-project/end-to-end/src/bench/client_flows/utils.ts new file mode 100644 index 000000000000..cbf90710ca94 --- /dev/null +++ b/yarn-project/end-to-end/src/bench/client_flows/utils.ts @@ -0,0 +1,267 @@ +import type { + ContractFunctionInteraction, + DeployMethod, + DeployOptions, + Logger, + ProfileMethodOptions, +} from '@aztec/aztec.js'; +import { createLogger } from '@aztec/foundation/log'; +import { type PrivateExecutionStep, serializePrivateExecutionSteps } from '@aztec/stdlib/kernel'; +import type { ProvingTimings, SimulationTimings } from '@aztec/stdlib/tx'; + +import assert from 'node:assert'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import type { GithubActionBenchmarkResult } from '../utils.js'; + +const logger = createLogger('bench:profile_capture'); + +const logLevel = ['silent', 'fatal', 'error', 'warn', 'info', 'verbose', 'debug', 'trace'] as const; +type LogLevel = (typeof logLevel)[number]; + +export type Log = { + type: LogLevel; + timestamp: number; + prefix: string; + message: string; + data: any; +}; + +const GATE_TYPES = [ + 'ecc_op', + 'busread', + 'lookup', + 'pub_inputs', + 'arithmetic', + 'delta_range', + 'elliptic', + 'aux', + 'poseidon2_external', + 'poseidon2_internal', + 'overflow', +] as const; + +type GateType = (typeof GATE_TYPES)[number]; + +type StructuredTrace = { + [k in GateType]: number; +}; + +export class ProxyLogger { + private static instance: ProxyLogger; + private logs: Log[] = []; + + private constructor() {} + + static create() { + ProxyLogger.instance = new ProxyLogger(); + } + + static getInstance() { + return ProxyLogger.instance; + } + + createLogger(prefix: string): Logger { + return new Proxy(createLogger(prefix), { + get: (target: Logger, prop: keyof Logger) => { + if (logLevel.includes(prop as (typeof logLevel)[number])) { + return function (this: Logger, ...data: Parameters) { + const loggingFn = prop as LogLevel; + const args = [loggingFn, prefix, ...data] as Parameters; + ProxyLogger.getInstance().handleLog(...args); + target[loggingFn].call(this, ...[data[0], data[1]]); + }; + } else { + return target[prop]; + } + }, + }); + } + + private handleLog(type: (typeof logLevel)[number], prefix: string, message: string, data: any) { + this.logs.unshift({ type, prefix, message, data, timestamp: Date.now() }); + } + + public flushLogs() { + this.logs = []; + } + + public getLogs() { + return this.logs; + } +} + +export type ProverType = 'wasm' | 'native'; + +type Step = Pick & { time: number; accGateCount?: number }; + +type ClientFlowBenchmark = { + name: string; + timings: Omit & { witgen: number }; + maxMemory: number; + proverType: ProverType; + minimumTrace: StructuredTrace; + totalGateCount: number; + steps: Step[]; + error: string | undefined; +}; + +function getMinimumTrace(logs: Log[]): StructuredTrace { + const minimumMessage = 'Minimum required block sizes for structured trace'; + const minimumMessageIndex = logs.findIndex(log => log.message.includes(minimumMessage)); + const candidateLogs = logs.slice(minimumMessageIndex - GATE_TYPES.length, minimumMessageIndex + 5); + const traceLogs = candidateLogs + .filter(log => GATE_TYPES.some(type => log.message.includes(type))) + .map(log => log.message.split(/\t|\n/)) + .flat() + .map(log => log.replace(/\(mem: .*\)/, '').trim()) + .filter(Boolean); + + const traceSizes = traceLogs.map(log => { + const [gateType, gateSizeStr] = log + .replace(/\n.*\)$/, '') + .replace(/bb - /, '') + .split(':') + .map(s => s.trim()); + const gateSize = parseInt(gateSizeStr); + assert(GATE_TYPES.includes(gateType as GateType), `Gate type ${gateType} is not recognized`); + return { [gateType]: gateSize }; + }); + + assert(traceSizes.length === GATE_TYPES.length, 'Decoded trace sizes do not match expected amount of gate types'); + return traceSizes.reduce((acc, curr) => ({ ...acc, ...curr }), {}) as StructuredTrace; +} + +function getMaxMemory(logs: Log[]): number { + const candidateLogs = logs.slice(0, 100).filter(log => /\(mem: .*MiB\)/.test(log.message)); + const usage = candidateLogs.map(log => { + const memStr = log ? log.message.slice(log.message.indexOf('(mem: ') + 6, log.message.indexOf('MiB') - 3) : ''; + return memStr ? parseInt(memStr) : 0; + }); + return Math.max(...usage); +} + +export function generateBenchmark( + flow: string, + logs: Log[], + timings: ProvingTimings | SimulationTimings, + privateExecutionSteps: PrivateExecutionStep[], + proverType: ProverType, + error: string | undefined, +): ClientFlowBenchmark { + const minimumTrace = getMinimumTrace(logs); + const maxMemory = getMaxMemory(logs); + + const steps = privateExecutionSteps.reduce((acc, step, i) => { + const previousAccGateCount = i === 0 ? 0 : acc[i - 1].accGateCount!; + return [ + ...acc, + { + functionName: step.functionName, + gateCount: step.gateCount, + accGateCount: previousAccGateCount + step.gateCount!, + time: step.timings.witgen, + }, + ]; + }, []); + const totalGateCount = steps[steps.length - 1].accGateCount; + return { + name: flow, + timings: { + total: timings.total, + sync: timings.sync!, + proving: (timings as ProvingTimings).proving, + unaccounted: timings.unaccounted, + witgen: timings.perFunction.reduce((acc, fn) => acc + fn.time, 0), + }, + maxMemory, + proverType, + minimumTrace, + totalGateCount: totalGateCount!, + steps, + error, + }; +} + +export function convertProfileToGHBenchmark(benchmark: ClientFlowBenchmark): GithubActionBenchmarkResult[] { + return [ + { + name: 'witgen', + value: benchmark.timings.witgen, + unit: 'ms', + }, + { + name: 'proving', + value: benchmark.timings.proving!, + unit: 'ms', + }, + { + name: 'total', + value: benchmark.timings.total, + unit: 'ms', + }, + { + name: 'sync', + value: benchmark.timings.sync!, + unit: 'ms', + }, + { + name: 'unaccounted', + value: benchmark.timings.unaccounted, + unit: 'ms', + }, + { + name: 'max_memory', + value: benchmark.maxMemory, + unit: 'MiB', + }, + { + name: 'total_gate_count', + value: benchmark.totalGateCount, + unit: 'gates', + }, + ]; +} + +export async function captureProfile( + label: string, + interaction: ContractFunctionInteraction | DeployMethod, + opts?: Omit, + expectedSteps?: number, +) { + // Make sure the proxy logger starts from a clean slate + ProxyLogger.getInstance().flushLogs(); + const result = await interaction.profile({ ...opts, profileMode: 'full', skipProofGeneration: false }); + const logs = ProxyLogger.getInstance().getLogs(); + if (expectedSteps !== undefined && result.executionSteps.length !== expectedSteps) { + throw new Error(`Expected ${expectedSteps} execution steps, got ${result.executionSteps.length}`); + } + const benchmark = generateBenchmark(label, logs, result.timings, result.executionSteps, 'wasm', undefined); + + const ivcFolder = process.env.CAPTURE_IVC_FOLDER; + if (ivcFolder) { + logger.info(`Capturing client ivc execution profile for ${label}`); + + const resultsDirectory = join(ivcFolder, label); + logger.info(`Writing private execution steps to ${resultsDirectory}`); + await mkdir(resultsDirectory, { recursive: true }); + // Write the client IVC files read by the prover. + const ivcInputsPath = join(resultsDirectory, 'ivc-inputs.msgpack'); + await writeFile(ivcInputsPath, serializePrivateExecutionSteps(result.executionSteps)); + await writeFile(join(resultsDirectory, 'logs.json'), JSON.stringify(logs, null, 2)); + await writeFile(join(resultsDirectory, 'benchmark.json'), JSON.stringify(benchmark, null, 2)); + logger.info(`Wrote private execution steps to ${resultsDirectory}`); + } + + const benchOutput = process.env.BENCH_OUTPUT; + if (benchOutput) { + await mkdir(benchOutput, { recursive: true }); + const ghBenchmark = convertProfileToGHBenchmark(benchmark); + const benchFile = join(benchOutput, `${label}.bench.json`); + await writeFile(benchFile, JSON.stringify(ghBenchmark)); + logger.info(`Wrote benchmark to ${benchFile}`); + } + + return result; +} diff --git a/yarn-project/end-to-end/src/bench/utils.ts b/yarn-project/end-to-end/src/bench/utils.ts index d2ca0e748b4d..41fdba0c1a71 100644 --- a/yarn-project/end-to-end/src/bench/utils.ts +++ b/yarn-project/end-to-end/src/bench/utils.ts @@ -53,7 +53,7 @@ type MetricFilter = { }; // See https://github.com/benchmark-action/github-action-benchmark/blob/e3c661617bc6aa55f26ae4457c737a55545a86a4/src/extract.ts#L659-L670 -type GithubActionBenchmarkResult = { +export type GithubActionBenchmarkResult = { name: string; value: number; range?: string; diff --git a/yarn-project/end-to-end/src/e2e_amm.test.ts b/yarn-project/end-to-end/src/e2e_amm.test.ts index 22a40717cda0..8c4274a8603d 100644 --- a/yarn-project/end-to-end/src/e2e_amm.test.ts +++ b/yarn-project/end-to-end/src/e2e_amm.test.ts @@ -6,7 +6,6 @@ import { jest } from '@jest/globals'; import { deployToken, mintTokensToPrivate } from './fixtures/token_utils.js'; import { setup } from './fixtures/utils.js'; -import { capturePrivateExecutionStepsIfEnvSet } from './shared/capture_private_execution_steps.js'; const TIMEOUT = 120_000; @@ -129,7 +128,6 @@ describe('AMM', () => { .withWallet(liquidityProvider) .methods.add_liquidity(amount0Max, amount1Max, amount0Min, amount1Min, nonceForAuthwits) .with({ authWitnesses: [token0Authwit, token1Authwit] }); - await capturePrivateExecutionStepsIfEnvSet('amm-add-liquidity', addLiquidityInteraction); await addLiquidityInteraction.send().wait(); const ammBalancesAfter = await getAmmBalances(); @@ -240,7 +238,6 @@ describe('AMM', () => { .withWallet(swapper) .methods.swap_exact_tokens_for_tokens(token0.address, token1.address, amountIn, amountOutMin, nonceForAuthwits) .with({ authWitnesses: [swapAuthwit] }); - await capturePrivateExecutionStepsIfEnvSet('amm-swap-exact-tokens', swapExactTokensInteraction); await swapExactTokensInteraction.send().wait(); // We know exactly how many tokens we're supposed to get because we know nobody else interacted with the AMM diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_private.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_private.test.ts index 6c03bb75548b..bfbabaf3046d 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_private.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_private.test.ts @@ -1,7 +1,6 @@ import { Fr, computeAuthWitMessageHash } from '@aztec/aztec.js'; import { DUPLICATE_NULLIFIER_ERROR } from '../fixtures/fixtures.js'; -import { capturePrivateExecutionStepsIfEnvSet } from '../shared/capture_private_execution_steps.js'; import { BlacklistTokenContractTest } from './blacklist_token_contract_test.js'; describe('e2e_blacklist_token_contract transfer private', () => { @@ -32,7 +31,6 @@ describe('e2e_blacklist_token_contract transfer private', () => { const tokenTransferInteraction = asset .withWallet(wallets[0]) .methods.transfer(wallets[0].getAddress(), wallets[1].getAddress(), amount, 0); - await capturePrivateExecutionStepsIfEnvSet('token-transfer', tokenTransferInteraction); await tokenTransferInteraction.send().wait(); tokenSim.transferPrivate(wallets[0].getAddress(), wallets[1].getAddress(), amount); }); diff --git a/yarn-project/end-to-end/src/e2e_nft.test.ts b/yarn-project/end-to-end/src/e2e_nft.test.ts index 973323ce3e57..59387b9c607d 100644 --- a/yarn-project/end-to-end/src/e2e_nft.test.ts +++ b/yarn-project/end-to-end/src/e2e_nft.test.ts @@ -4,7 +4,6 @@ import { NFTContract } from '@aztec/noir-contracts.js/NFT'; import { jest } from '@jest/globals'; import { setup } from './fixtures/utils.js'; -import { capturePrivateExecutionStepsIfEnvSet } from './shared/capture_private_execution_steps.js'; const TIMEOUT = 120_000; @@ -51,7 +50,6 @@ describe('NFT', () => { const nftContractAsMinter = await NFTContract.at(nftContractAddress, minterWallet); const nftMintInteraction = nftContractAsMinter.methods.mint(user1Wallet.getAddress(), TOKEN_ID); - await capturePrivateExecutionStepsIfEnvSet('nft-mint', nftMintInteraction); await nftMintInteraction.send().wait(); const ownerAfterMint = await nftContractAsMinter.methods.owner_of(TOKEN_ID).simulate(); @@ -80,7 +78,6 @@ describe('NFT', () => { TOKEN_ID, 0, ); - await capturePrivateExecutionStepsIfEnvSet('nft-transfer-in-private', nftTransferInteraction); await nftTransferInteraction.send().wait(); const user1Nfts = await getPrivateNfts(user1Wallet.getAddress()); diff --git a/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts b/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts index 8af2fecb2fee..ec00368f305d 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts @@ -286,7 +286,11 @@ describe('e2e_p2p_add_rollup', () => { ) => { // Bridge assets into the rollup, and consume the message. // We are doing some of the things that are in the crosschain harness, but we don't actually want the full thing - const pxeService = await createPXEService(node, { ...getPXEServiceConfig(), proverEnabled: false }, true); + const pxeService = await createPXEService( + node, + { ...getPXEServiceConfig(), proverEnabled: false }, + { useLogSuffix: true }, + ); await deployFundedSchnorrAccount(pxeService, aliceAccount, undefined, undefined); const alice = await getSchnorrWalletWithSecretKey( diff --git a/yarn-project/end-to-end/src/e2e_p2p/shared.ts b/yarn-project/end-to-end/src/e2e_p2p/shared.ts index 855817727719..3045772ec65c 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/shared.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/shared.ts @@ -48,7 +48,7 @@ export const createPXEServiceAndSubmitTransactions = async ( ): Promise => { const rpcConfig = getRpcConfig(); rpcConfig.proverEnabled = false; - const pxeService = await createPXEService(node, rpcConfig, true); + const pxeService = await createPXEService(node, rpcConfig, { useLogSuffix: true }); const account = await getSchnorrAccount( pxeService, @@ -71,7 +71,7 @@ export async function createPXEServiceAndPrepareTransactions( ): Promise<{ pxeService: PXEService; txs: ProvenTx[]; node: AztecNodeService }> { const rpcConfig = getRpcConfig(); rpcConfig.proverEnabled = false; - const pxe = await createPXEService(node, rpcConfig, true); + const pxe = await createPXEService(node, rpcConfig, { useLogSuffix: true }); const account = await getSchnorrAccount(pxe, fundedAccount.secret, fundedAccount.signingKey, fundedAccount.salt); await account.register(); diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index dbd4fd826272..724379e719d9 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -176,7 +176,7 @@ export async function setupPXEService( aztecNode, simulationProviderWithRecorder, pxeServiceConfig, - useLogSuffix, + { useLogSuffix }, ); const teardown = async () => { diff --git a/yarn-project/end-to-end/src/shared/capture_private_execution_steps.ts b/yarn-project/end-to-end/src/shared/capture_private_execution_steps.ts deleted file mode 100644 index d9a17b115e10..000000000000 --- a/yarn-project/end-to-end/src/shared/capture_private_execution_steps.ts +++ /dev/null @@ -1,68 +0,0 @@ -/** - * This module exposes the ability to capture the private exection steps that go into our "Client IVC" prover. - * These are used for debugging and benchmarking barretenberg (the prover component). - */ -import type { - ContractFunctionInteraction, - DeployMethod, - DeployOptions, - ProfileMethodOptions, -} from '@aztec/aztec.js/contracts'; -import { createLogger } from '@aztec/foundation/log'; -import { serializePrivateExecutionSteps } from '@aztec/stdlib/kernel'; - -import { promises as fs } from 'fs'; -import path from 'path'; - -const logger = createLogger('e2e:capture-private-execution-steps'); - -export async function capturePrivateExecutionStepsIfEnvSet( - label: string, - interaction: ContractFunctionInteraction | DeployMethod, - opts?: Omit, - expectedSteps?: number, -) { - // Not included in env_var.ts as internal to e2e tests. - const ivcFolder = process.env.CAPTURE_IVC_FOLDER; - if (!ivcFolder) { - return; - } - const profileMode = ['execution-steps', 'full'].includes(process.env.PROFILE_MODE ?? '') - ? (process.env.PROFILE_MODE as 'full' | 'execution-steps') - : 'execution-steps'; - logger.info(`Capturing client ivc execution profile for ${label} in mode ${profileMode}`); - const result = await interaction.profile({ ...opts, profileMode }); - if (expectedSteps !== undefined && result.executionSteps.length !== expectedSteps) { - throw new Error(`Expected ${expectedSteps} execution steps, got ${result.executionSteps.length}`); - } - const resultsDirectory = path.join(ivcFolder, label); - logger.info(`Writing private execution steps to ${resultsDirectory}`); - await fs.mkdir(resultsDirectory, { recursive: true }); - // Write the client IVC files read by the prover. - const ivcInputsPath = path.join(resultsDirectory, 'ivc-inputs.msgpack'); - await fs.writeFile(ivcInputsPath, serializePrivateExecutionSteps(result.executionSteps)); - if (profileMode === 'full') { - // If we have gate counts, write the steps in human-readable format. - await fs.writeFile( - path.join(resultsDirectory, 'profile.json'), - JSON.stringify( - { - timings: result.timings, - steps: result.executionSteps.map(step => ({ - fnName: step.functionName, - gateCount: step.gateCount, - timings: step.timings, - })), - }, - null, - 2, - ), - ); - // In full mode, we also write the raw witnesses in a more human-readable format. - await fs.writeFile( - path.join(resultsDirectory, 'witnesses.json'), - JSON.stringify(result.executionSteps.map(step => Object.fromEntries(step.witness))), - ); - } - logger.info(`Wrote private execution steps to ${resultsDirectory}`); -} diff --git a/yarn-project/pxe/src/entrypoints/client/bundle/utils.ts b/yarn-project/pxe/src/entrypoints/client/bundle/utils.ts index 6d951ec32863..dcd5565a4f96 100644 --- a/yarn-project/pxe/src/entrypoints/client/bundle/utils.ts +++ b/yarn-project/pxe/src/entrypoints/client/bundle/utils.ts @@ -1,4 +1,5 @@ import { BBWASMBundlePrivateKernelProver } from '@aztec/bb-prover/client/wasm/bundle'; +import { randomBytes } from '@aztec/foundation/crypto'; import { createLogger } from '@aztec/foundation/log'; import { createStore } from '@aztec/kv-store/indexeddb'; import { BundledProtocolContractsProvider } from '@aztec/protocol-contracts/providers/bundle'; @@ -7,7 +8,7 @@ import type { AztecNode } from '@aztec/stdlib/interfaces/client'; import type { PXEServiceConfig } from '../../../config/index.js'; import { PXEService } from '../../../pxe_service/pxe_service.js'; -import type { PXECreationOptions } from '../pxe_creation_options.js'; +import type { PXECreationOptions } from '../../pxe_creation_options.js'; /** * Create and start an PXEService instance with the given AztecNode. @@ -24,6 +25,15 @@ export async function createPXEService( config: PXEServiceConfig, options: PXECreationOptions = { loggers: {} }, ) { + const logSuffix = + typeof options.useLogSuffix === 'boolean' + ? options.useLogSuffix + ? randomBytes(3).toString('hex') + : undefined + : options.useLogSuffix; + + const loggers = options.loggers ?? {}; + const l1Contracts = await aztecNode.getL1ContractAddresses(); const configWithContracts = { ...config, @@ -31,21 +41,19 @@ export async function createPXEService( l2BlockBatchSize: 200, } as PXEServiceConfig; - const store = await createStore( - 'pxe_data', - configWithContracts, - options.loggers.store ?? createLogger('pxe:data:indexeddb'), - ); + const storeLogger = loggers.store ? loggers.store : createLogger('pxe:data:idb' + (logSuffix ? `:${logSuffix}` : '')); + + const store = options.store ?? (await createStore('pxe_data', configWithContracts, storeLogger)); const simulationProvider = new WASMSimulator(); - const prover = - options.prover ?? - new BBWASMBundlePrivateKernelProver( - simulationProvider, - 16, - options.loggers.prover ?? createLogger('bb:wasm:bundle'), - ); + const proverLogger = loggers.prover + ? loggers.prover + : createLogger('pxe:bb:wasm:bundle' + (logSuffix ? `:${logSuffix}` : '')); + + const prover = options.prover ?? new BBWASMBundlePrivateKernelProver(simulationProvider, 16, proverLogger); const protocolContractsProvider = new BundledProtocolContractsProvider(); + + const pxeLogger = loggers.pxe ? loggers.pxe : createLogger('pxe:service' + (logSuffix ? `:${logSuffix}` : '')); const pxe = await PXEService.create( aztecNode, store, @@ -53,7 +61,7 @@ export async function createPXEService( simulationProvider, protocolContractsProvider, config, - options.loggers.pxe ?? createLogger('pxe:service'), + pxeLogger, ); return pxe; } diff --git a/yarn-project/pxe/src/entrypoints/client/lazy/utils.ts b/yarn-project/pxe/src/entrypoints/client/lazy/utils.ts index 7882d612fa91..90412821ec74 100644 --- a/yarn-project/pxe/src/entrypoints/client/lazy/utils.ts +++ b/yarn-project/pxe/src/entrypoints/client/lazy/utils.ts @@ -1,4 +1,5 @@ import { BBWASMLazyPrivateKernelProver } from '@aztec/bb-prover/client/wasm/lazy'; +import { randomBytes } from '@aztec/foundation/crypto'; import { createLogger } from '@aztec/foundation/log'; import { createStore } from '@aztec/kv-store/indexeddb'; import { LazyProtocolContractsProvider } from '@aztec/protocol-contracts/providers/lazy'; @@ -7,7 +8,7 @@ import type { AztecNode } from '@aztec/stdlib/interfaces/client'; import type { PXEServiceConfig } from '../../../config/index.js'; import { PXEService } from '../../../pxe_service/pxe_service.js'; -import type { PXECreationOptions } from '../pxe_creation_options.js'; +import type { PXECreationOptions } from '../../pxe_creation_options.js'; /** * Create and start an PXEService instance with the given AztecNode. @@ -23,6 +24,13 @@ export async function createPXEService( config: PXEServiceConfig, options: PXECreationOptions = { loggers: {} }, ) { + const logSuffix = + typeof options.useLogSuffix === 'boolean' + ? options.useLogSuffix + ? randomBytes(3).toString('hex') + : undefined + : options.useLogSuffix; + const l1Contracts = await aztecNode.getL1ContractAddresses(); const configWithContracts = { ...config, @@ -30,17 +38,22 @@ export async function createPXEService( l1Contracts, } as PXEServiceConfig; - const store = await createStore( - 'pxe_data', - configWithContracts, - options.loggers.store ?? createLogger('pxe:data:indexeddb'), - ); + const loggers = options.loggers ?? {}; + + const storeLogger = loggers.store ? loggers.store : createLogger('pxe:data:idb' + (logSuffix ? `:${logSuffix}` : '')); + + const store = options.store ?? (await createStore('pxe_data', configWithContracts, storeLogger)); const simulationProvider = new WASMSimulator(); - const prover = - options.prover ?? - new BBWASMLazyPrivateKernelProver(simulationProvider, 16, options.loggers.prover ?? createLogger('bb:wasm:lazy')); + const proverLogger = loggers.prover + ? loggers.prover + : createLogger('pxe:bb:wasm:bundle' + (logSuffix ? `:${logSuffix}` : '')); + + const prover = options.prover ?? new BBWASMLazyPrivateKernelProver(simulationProvider, 16, proverLogger); + const protocolContractsProvider = new LazyProtocolContractsProvider(); + + const pxeLogger = loggers.pxe ? loggers.pxe : createLogger('pxe:service' + (logSuffix ? `:${logSuffix}` : '')); const pxe = await PXEService.create( aztecNode, store, @@ -48,7 +61,7 @@ export async function createPXEService( simulationProvider, protocolContractsProvider, config, - options.loggers.pxe ?? createLogger('pxe:service'), + pxeLogger, ); return pxe; } diff --git a/yarn-project/pxe/src/entrypoints/client/pxe_creation_options.ts b/yarn-project/pxe/src/entrypoints/pxe_creation_options.ts similarity index 51% rename from yarn-project/pxe/src/entrypoints/client/pxe_creation_options.ts rename to yarn-project/pxe/src/entrypoints/pxe_creation_options.ts index 5f76dbffb58c..4148dcce5717 100644 --- a/yarn-project/pxe/src/entrypoints/client/pxe_creation_options.ts +++ b/yarn-project/pxe/src/entrypoints/pxe_creation_options.ts @@ -1,7 +1,10 @@ import type { Logger } from '@aztec/foundation/log'; +import type { AztecAsyncKVStore } from '@aztec/kv-store'; import type { PrivateKernelProver } from '@aztec/stdlib/interfaces/client'; export type PXECreationOptions = { - loggers: { store?: Logger; pxe?: Logger; prover?: Logger }; + loggers?: { store?: Logger; pxe?: Logger; prover?: Logger }; + useLogSuffix?: boolean | string; prover?: PrivateKernelProver; + store?: AztecAsyncKVStore; }; diff --git a/yarn-project/pxe/src/entrypoints/server/utils.ts b/yarn-project/pxe/src/entrypoints/server/utils.ts index f29117049f2e..3402d8d63f0a 100644 --- a/yarn-project/pxe/src/entrypoints/server/utils.ts +++ b/yarn-project/pxe/src/entrypoints/server/utils.ts @@ -1,8 +1,7 @@ import { BBNativePrivateKernelProver } from '@aztec/bb-prover/client/native'; import { BBWASMBundlePrivateKernelProver } from '@aztec/bb-prover/client/wasm/bundle'; import { randomBytes } from '@aztec/foundation/crypto'; -import { createLogger } from '@aztec/foundation/log'; -import type { AztecAsyncKVStore } from '@aztec/kv-store'; +import { type Logger, createLogger } from '@aztec/foundation/log'; import { BundledProtocolContractsProvider } from '@aztec/protocol-contracts/providers/bundle'; import { type SimulationProvider, WASMSimulator } from '@aztec/simulator/client'; import { SimulationProviderRecorderWrapper } from '@aztec/simulator/testing'; @@ -11,6 +10,7 @@ import type { AztecNode } from '@aztec/stdlib/interfaces/client'; import type { PXEServiceConfig } from '../../config/index.js'; import { PXEService } from '../../pxe_service/pxe_service.js'; import { PXE_DATA_SCHEMA_VERSION } from '../../storage/index.js'; +import type { PXECreationOptions } from '../pxe_creation_options.js'; /** * Create and start an PXEService instance with the given AztecNode and config. @@ -23,12 +23,11 @@ import { PXE_DATA_SCHEMA_VERSION } from '../../storage/index.js'; export function createPXEService( aztecNode: AztecNode, config: PXEServiceConfig, - useLogSuffix: string | boolean | undefined = undefined, - store?: AztecAsyncKVStore, + options: PXECreationOptions = { loggers: {} }, ) { const simulationProvider = new WASMSimulator(); const simulationProviderWithRecorder = new SimulationProviderRecorderWrapper(simulationProvider); - return createPXEServiceWithSimulationProvider(aztecNode, simulationProviderWithRecorder, config, useLogSuffix, store); + return createPXEServiceWithSimulationProvider(aztecNode, simulationProviderWithRecorder, config, options); } /** @@ -44,11 +43,15 @@ export async function createPXEServiceWithSimulationProvider( aztecNode: AztecNode, simulationProvider: SimulationProvider, config: PXEServiceConfig, - useLogSuffix: string | boolean | undefined = undefined, - store?: AztecAsyncKVStore, + options: PXECreationOptions = { loggers: {} }, ) { const logSuffix = - typeof useLogSuffix === 'boolean' ? (useLogSuffix ? randomBytes(3).toString('hex') : undefined) : useLogSuffix; + typeof options.useLogSuffix === 'boolean' + ? options.useLogSuffix + ? randomBytes(3).toString('hex') + : undefined + : options.useLogSuffix; + const loggers = options.loggers ?? {}; const l1Contracts = await aztecNode.getL1ContractAddresses(); const configWithContracts = { @@ -57,34 +60,41 @@ export async function createPXEServiceWithSimulationProvider( l2BlockBatchSize: 200, } as PXEServiceConfig; - if (!store) { + if (!options.store) { // TODO once https://github.com/AztecProtocol/aztec-packages/issues/13656 is fixed, we can remove this and always // import the lmdb-v2 version const { createStore } = await import('@aztec/kv-store/lmdb-v2'); - store = await createStore('pxe_data', PXE_DATA_SCHEMA_VERSION, configWithContracts, createLogger('pxe:data:lmdb')); + const storeLogger = loggers.store + ? loggers.store + : createLogger('pxe:data:lmdb' + (logSuffix ? `:${logSuffix}` : '')); + options.store = await createStore('pxe_data', PXE_DATA_SCHEMA_VERSION, configWithContracts, storeLogger); } + const proverLogger = loggers.prover + ? loggers.prover + : createLogger('pxe:bb:native' + (logSuffix ? `:${logSuffix}` : '')); - const prover = await createProver(config, simulationProvider, logSuffix); + const prover = await createProver(config, simulationProvider, proverLogger); const protocolContractsProvider = new BundledProtocolContractsProvider(); + + const pxeLogger = loggers.pxe ? loggers.pxe : createLogger('pxe:service' + (logSuffix ? `:${logSuffix}` : '')); const pxe = await PXEService.create( aztecNode, - store, + options.store, prover, simulationProvider, protocolContractsProvider, config, - logSuffix, + pxeLogger, ); return pxe; } -function createProver(config: PXEServiceConfig, simulationProvider: SimulationProvider, logSuffix?: string) { +function createProver(config: PXEServiceConfig, simulationProvider: SimulationProvider, logger?: Logger) { if (!config.bbBinaryPath || !config.bbWorkingDirectory) { - return new BBWASMBundlePrivateKernelProver(simulationProvider, 16); + return new BBWASMBundlePrivateKernelProver(simulationProvider, 16, logger); } else { const bbConfig = config as Required> & PXEServiceConfig; - const log = createLogger('pxe:bb-native-prover' + (logSuffix ? `:${logSuffix}` : '')); - return BBNativePrivateKernelProver.new({ bbSkipCleanup: false, ...bbConfig }, simulationProvider, log); + return BBNativePrivateKernelProver.new({ bbSkipCleanup: false, ...bbConfig }, simulationProvider, logger); } } diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index d21bc61ba2c7..a3642289be7d 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -329,11 +329,13 @@ __metadata: "@jest/globals": "npm:^29.5.0" "@types/jest": "npm:^29.5.0" "@types/node": "npm:^22.15.17" + "@types/pidusage": "npm:^2.0.5" "@types/source-map-support": "npm:^0.5.10" commander: "npm:^12.1.0" jest: "npm:^29.5.0" jest-mock-extended: "npm:^3.0.3" pako: "npm:^2.1.0" + pidusage: "npm:^4.0.1" source-map-support: "npm:^0.5.21" ts-node: "npm:^10.9.1" tslib: "npm:^2.4.0" @@ -6372,6 +6374,13 @@ __metadata: languageName: node linkType: hard +"@types/pidusage@npm:^2.0.5": + version: 2.0.5 + resolution: "@types/pidusage@npm:2.0.5" + checksum: 10/24188bf108b9b5a2ccb16155a492ff1cca7d7bf07aa4e2649cce421e0940dd5b0cd06baa48da9b8bde46343463fc500a7070c0788a0ed3f3f57d3e3bcf4ac64d + languageName: node + linkType: hard + "@types/qs@npm:*": version: 6.9.15 resolution: "@types/qs@npm:6.9.15" @@ -18395,6 +18404,15 @@ __metadata: languageName: node linkType: hard +"pidusage@npm:^4.0.1": + version: 4.0.1 + resolution: "pidusage@npm:4.0.1" + dependencies: + safe-buffer: "npm:^5.2.1" + checksum: 10/ce791293a32c3c9a89ffe1033d0fd132606278adc68cbfd24c201562edec1b8fdc6df86e89fda1e0f9be7906449cc6173aaade7853c6dec22080258bc694e548 + languageName: node + linkType: hard + "pinkie-promise@npm:^2.0.0": version: 2.0.1 resolution: "pinkie-promise@npm:2.0.1" From 67a3f74f83d11b1cc2d0236fde5a7d1f888285d4 Mon Sep 17 00:00:00 2001 From: thunkar Date: Mon, 26 May 2025 06:14:39 +0000 Subject: [PATCH 2/8] sad face --- bootstrap.sh | 3 +- yarn-project/end-to-end/bootstrap.sh | 8 +++- .../client_flows/account_deployments.test.ts | 2 +- .../src/bench/client_flows/amm.test.ts | 2 +- .../client_flows/{utils.ts => benchmark.ts} | 42 ++++++++++++------- .../src/bench/client_flows/bridging.test.ts | 2 +- .../client_flows/client_flows_benchmark.ts | 2 +- .../src/bench/client_flows/data_extractor.ts | 2 +- .../bench/client_flows/deployments.test.ts | 2 +- .../src/bench/client_flows/transfers.test.ts | 2 +- 10 files changed, 44 insertions(+), 23 deletions(-) rename yarn-project/end-to-end/src/bench/client_flows/{utils.ts => benchmark.ts} (94%) diff --git a/bootstrap.sh b/bootstrap.sh index 5840799aac1e..408786682ffc 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -253,7 +253,8 @@ function build_bench { return fi parallel --line-buffer --tag --halt now,fail=1 'denoise "{}/bootstrap.sh build_bench"' ::: \ - barretenberg/cpp + barretenberg/cpp \ + yarn-project/end-to-end } export -f build_bench diff --git a/yarn-project/end-to-end/bootstrap.sh b/yarn-project/end-to-end/bootstrap.sh index 2f4a480998d2..88091f00d35e 100755 --- a/yarn-project/end-to-end/bootstrap.sh +++ b/yarn-project/end-to-end/bootstrap.sh @@ -61,7 +61,13 @@ function bench_cmds { echo "$hash:ISOLATE=1:NAME=bench_build_block BENCH_OUTPUT=bench-out/build-block.bench.json yarn-project/end-to-end/scripts/run_test.sh simple bench_build_block" for client_flow in client_flows/bridging client_flows/deployments client_flows/amm client_flows/account_deployments client_flows/transfers; do - echo "$hash:ISOLATE=1:CPUS=8:NAME=$client_flow REAL_PROOFS=1 BENCHMARK_CONFIG=key_flows LOG_LEVEL=error BENCH_OUTPUT=bench-out/ yarn-project/end-to-end/scripts/run_test.sh simple $client_flow" + echo "$hash:ISOLATE=1:CPUS=8:NAME=$client_flow BENCHMARK_CONFIG=key_flows LOG_LEVEL=error BENCH_OUTPUT=bench-out/ yarn-project/end-to-end/scripts/run_test.sh simple $client_flow" + done + + for dir in $bench_fixtures_dir/*; do + for runtime in native wasm; do + echo "$hash:CPUS=8 barretenberg/cpp/scripts/ci_benchmark_ivc_flows.sh $runtime ../../yarn-project/end-to-end/$dir" + done done } diff --git a/yarn-project/end-to-end/src/bench/client_flows/account_deployments.test.ts b/yarn-project/end-to-end/src/bench/client_flows/account_deployments.test.ts index 650e7ab9365c..957bd741b58d 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/account_deployments.test.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/account_deployments.test.ts @@ -4,8 +4,8 @@ import type { SponsoredFPCContract } from '@aztec/noir-contracts.js/SponsoredFPC import { jest } from '@jest/globals'; +import { captureProfile } from './benchmark.js'; import { type AccountType, type BenchmarkingFeePaymentMethod, ClientFlowsBenchmark } from './client_flows_benchmark.js'; -import { captureProfile } from './utils.js'; jest.setTimeout(300_000); diff --git a/yarn-project/end-to-end/src/bench/client_flows/amm.test.ts b/yarn-project/end-to-end/src/bench/client_flows/amm.test.ts index dbc2c27991c4..3eea2a1390a5 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/amm.test.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/amm.test.ts @@ -8,8 +8,8 @@ import { TokenContract } from '@aztec/noir-contracts.js/Token'; import { jest } from '@jest/globals'; import { mintNotes } from '../../fixtures/token_utils.js'; +import { captureProfile } from './benchmark.js'; import { type AccountType, type BenchmarkingFeePaymentMethod, ClientFlowsBenchmark } from './client_flows_benchmark.js'; -import { captureProfile } from './utils.js'; jest.setTimeout(900_000); diff --git a/yarn-project/end-to-end/src/bench/client_flows/utils.ts b/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts similarity index 94% rename from yarn-project/end-to-end/src/bench/client_flows/utils.ts rename to yarn-project/end-to-end/src/bench/client_flows/benchmark.ts index cbf90710ca94..652d6e501df3 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/utils.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts @@ -111,6 +111,7 @@ function getMinimumTrace(logs: Log[]): StructuredTrace { const minimumMessage = 'Minimum required block sizes for structured trace'; const minimumMessageIndex = logs.findIndex(log => log.message.includes(minimumMessage)); const candidateLogs = logs.slice(minimumMessageIndex - GATE_TYPES.length, minimumMessageIndex + 5); + const traceLogs = candidateLogs .filter(log => GATE_TYPES.some(type => log.message.includes(type))) .map(log => log.message.split(/\t|\n/)) @@ -150,8 +151,14 @@ export function generateBenchmark( proverType: ProverType, error: string | undefined, ): ClientFlowBenchmark { - const minimumTrace = getMinimumTrace(logs); - const maxMemory = getMaxMemory(logs); + let maxMemory = 0; + let minimumTrace: StructuredTrace; + try { + minimumTrace = getMinimumTrace(logs); + maxMemory = getMaxMemory(logs); + } catch (e) { + logger.warn(`Failed obtain minimum trace and max memory for ${flow}. Did you run with REAL_PROOFS=1?`); + } const steps = privateExecutionSteps.reduce((acc, step, i) => { const previousAccGateCount = i === 0 ? 0 : acc[i - 1].accGateCount!; @@ -177,7 +184,7 @@ export function generateBenchmark( }, maxMemory, proverType, - minimumTrace, + minimumTrace: minimumTrace!, totalGateCount: totalGateCount!, steps, error, @@ -185,17 +192,13 @@ export function generateBenchmark( } export function convertProfileToGHBenchmark(benchmark: ClientFlowBenchmark): GithubActionBenchmarkResult[] { - return [ + const benches = [ { name: 'witgen', value: benchmark.timings.witgen, unit: 'ms', }, - { - name: 'proving', - value: benchmark.timings.proving!, - unit: 'ms', - }, + { name: 'total', value: benchmark.timings.total, @@ -211,17 +214,28 @@ export function convertProfileToGHBenchmark(benchmark: ClientFlowBenchmark): Git value: benchmark.timings.unaccounted, unit: 'ms', }, - { - name: 'max_memory', - value: benchmark.maxMemory, - unit: 'MiB', - }, + { name: 'total_gate_count', value: benchmark.totalGateCount, unit: 'gates', }, ]; + if (benchmark.timings.proving) { + benches.push({ + name: 'proving', + value: benchmark.timings.proving, + unit: 'ms', + }); + } + if (benchmark.maxMemory) { + benches.push({ + name: 'max_memory', + value: benchmark.maxMemory, + unit: 'MiB', + }); + } + return benches; } export async function captureProfile( diff --git a/yarn-project/end-to-end/src/bench/client_flows/bridging.test.ts b/yarn-project/end-to-end/src/bench/client_flows/bridging.test.ts index a348f0b622b4..2107dfe4d20e 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/bridging.test.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/bridging.test.ts @@ -7,8 +7,8 @@ import { TokenContract } from '@aztec/noir-contracts.js/Token'; import { jest } from '@jest/globals'; import type { CrossChainTestHarness } from '../../shared/cross_chain_test_harness.js'; +import { captureProfile } from './benchmark.js'; import { type AccountType, type BenchmarkingFeePaymentMethod, ClientFlowsBenchmark } from './client_flows_benchmark.js'; -import { captureProfile } from './utils.js'; jest.setTimeout(300_000); diff --git a/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts b/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts index 4563713837b3..6bb1e1e9b498 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts @@ -48,8 +48,8 @@ import { FeeJuicePortalTestingHarnessFactory, type GasBridgingTestHarness, } from '../../shared/gas_portal_test_harness.js'; +import { ProxyLogger } from './benchmark.js'; import { type ClientFlowsConfig, FULL_FLOWS_CONFIG, KEY_FLOWS_CONFIG } from './config.js'; -import { ProxyLogger } from './utils.js'; const { E2E_DATA_PATH: dataPath, BENCHMARK_CONFIG } = process.env; diff --git a/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts b/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts index 3531ad9e1731..e674b121e7fe 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts @@ -11,7 +11,7 @@ import { Decoder } from 'msgpackr'; import { readFile, readdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { type Log, type ProverType, ProxyLogger, generateBenchmark } from './utils.js'; +import { type Log, type ProverType, ProxyLogger, generateBenchmark } from './benchmark.js'; type NativeProverConfig = { bbBinaryPath?: string; bbWorkingDirectory?: string }; diff --git a/yarn-project/end-to-end/src/bench/client_flows/deployments.test.ts b/yarn-project/end-to-end/src/bench/client_flows/deployments.test.ts index 27f91f7d72ef..4684569a939c 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/deployments.test.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/deployments.test.ts @@ -5,8 +5,8 @@ import { getContractClassFromArtifact } from '@aztec/stdlib/contract'; import { jest } from '@jest/globals'; +import { captureProfile } from './benchmark.js'; import { type AccountType, type BenchmarkingFeePaymentMethod, ClientFlowsBenchmark } from './client_flows_benchmark.js'; -import { captureProfile } from './utils.js'; jest.setTimeout(1_600_000); diff --git a/yarn-project/end-to-end/src/bench/client_flows/transfers.test.ts b/yarn-project/end-to-end/src/bench/client_flows/transfers.test.ts index 566495cd60af..4e553c530b50 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/transfers.test.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/transfers.test.ts @@ -7,8 +7,8 @@ import { TokenContract } from '@aztec/noir-contracts.js/Token'; import { jest } from '@jest/globals'; import { mintNotes } from '../../fixtures/token_utils.js'; +import { captureProfile } from './benchmark.js'; import { type AccountType, type BenchmarkingFeePaymentMethod, ClientFlowsBenchmark } from './client_flows_benchmark.js'; -import { captureProfile } from './utils.js'; jest.setTimeout(1_600_000); From 7262c6d5481c5eeb58e0615a2a019f5f85535f51 Mon Sep 17 00:00:00 2001 From: thunkar Date: Mon, 26 May 2025 06:32:25 +0000 Subject: [PATCH 3/8] unique names --- .../end-to-end/src/bench/client_flows/benchmark.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts b/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts index 652d6e501df3..c6d1c94f61ce 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts @@ -194,43 +194,43 @@ export function generateBenchmark( export function convertProfileToGHBenchmark(benchmark: ClientFlowBenchmark): GithubActionBenchmarkResult[] { const benches = [ { - name: 'witgen', + name: `${label}/witgen`, value: benchmark.timings.witgen, unit: 'ms', }, { - name: 'total', + name: `${label}/total`, value: benchmark.timings.total, unit: 'ms', }, { - name: 'sync', + name: `${label}/sync`, value: benchmark.timings.sync!, unit: 'ms', }, { - name: 'unaccounted', + name: `${label}/unaccounte`, value: benchmark.timings.unaccounted, unit: 'ms', }, { - name: 'total_gate_count', + name: `${label}/total_gate_count`, value: benchmark.totalGateCount, unit: 'gates', }, ]; if (benchmark.timings.proving) { benches.push({ - name: 'proving', + name: `${label}/proving`, value: benchmark.timings.proving, unit: 'ms', }); } if (benchmark.maxMemory) { benches.push({ - name: 'max_memory', + name: `${label}/max_memory`, value: benchmark.maxMemory, unit: 'MiB', }); From 55f37ae506c1c3453034547d95b841fc2e715970 Mon Sep 17 00:00:00 2001 From: thunkar Date: Mon, 26 May 2025 06:36:56 +0000 Subject: [PATCH 4/8] whoops --- .../end-to-end/src/bench/client_flows/benchmark.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts b/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts index c6d1c94f61ce..05fb0d7721be 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts @@ -194,43 +194,43 @@ export function generateBenchmark( export function convertProfileToGHBenchmark(benchmark: ClientFlowBenchmark): GithubActionBenchmarkResult[] { const benches = [ { - name: `${label}/witgen`, + name: `${benchmark.name}/witgen`, value: benchmark.timings.witgen, unit: 'ms', }, { - name: `${label}/total`, + name: `${benchmark.name}/total`, value: benchmark.timings.total, unit: 'ms', }, { - name: `${label}/sync`, + name: `${benchmark.name}/sync`, value: benchmark.timings.sync!, unit: 'ms', }, { - name: `${label}/unaccounte`, + name: `${benchmark.name}/unaccounte`, value: benchmark.timings.unaccounted, unit: 'ms', }, { - name: `${label}/total_gate_count`, + name: `${benchmark.name}/total_gate_count`, value: benchmark.totalGateCount, unit: 'gates', }, ]; if (benchmark.timings.proving) { benches.push({ - name: `${label}/proving`, + name: `${benchmark.name}/proving`, value: benchmark.timings.proving, unit: 'ms', }); } if (benchmark.maxMemory) { benches.push({ - name: `${label}/max_memory`, + name: `${benchmark.name}/max_memory`, value: benchmark.maxMemory, unit: 'MiB', }); From ef9ef2ab3242ef7d32897c57122e5f0dca358b8c Mon Sep 17 00:00:00 2001 From: thunkar Date: Mon, 26 May 2025 07:55:55 +0000 Subject: [PATCH 5/8] typo --- yarn-project/end-to-end/src/bench/client_flows/benchmark.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts b/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts index 05fb0d7721be..34a31b8b9780 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts @@ -210,7 +210,7 @@ export function convertProfileToGHBenchmark(benchmark: ClientFlowBenchmark): Git unit: 'ms', }, { - name: `${benchmark.name}/unaccounte`, + name: `${benchmark.name}/unaccounted`, value: benchmark.timings.unaccounted, unit: 'ms', }, From 00e9e8672ed0958d0cd1d473099ffd60f9a68d4d Mon Sep 17 00:00:00 2001 From: thunkar Date: Mon, 26 May 2025 09:02:59 +0000 Subject: [PATCH 6/8] fmt --- yarn-project/end-to-end/src/bench/client_flows/benchmark.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts b/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts index 34a31b8b9780..801009f81b78 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/benchmark.ts @@ -156,7 +156,7 @@ export function generateBenchmark( try { minimumTrace = getMinimumTrace(logs); maxMemory = getMaxMemory(logs); - } catch (e) { + } catch { logger.warn(`Failed obtain minimum trace and max memory for ${flow}. Did you run with REAL_PROOFS=1?`); } From 3a1f645c152c22eae9d14923a6ce969105f855c6 Mon Sep 17 00:00:00 2001 From: thunkar Date: Mon, 26 May 2025 10:20:31 +0000 Subject: [PATCH 7/8] check --- bootstrap.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index 408786682ffc..a3b116a5c7f8 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -195,6 +195,8 @@ function test { # and also that half the cpus are logical, not physical. echo "Gathering tests to run..." tests=$(test_cmds $@) + tests+=('build_bench') + # Note: Capturing strips last newline. The echo re-adds it. local num [ -z "$tests" ] && num=0 || num=$(echo "$tests" | wc -l) @@ -229,7 +231,6 @@ function build { release-image/bootstrap.sh spartan/bootstrap.sh aztec-up/bootstrap.sh - build_bench ) for project in "${serial_projects[@]}"; do From 523d16756116336e485213fe485a75e82110d8b9 Mon Sep 17 00:00:00 2001 From: thunkar Date: Mon, 26 May 2025 11:00:57 +0000 Subject: [PATCH 8/8] different approach --- bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index a3b116a5c7f8..81ed58c89ecf 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -195,7 +195,6 @@ function test { # and also that half the cpus are logical, not physical. echo "Gathering tests to run..." tests=$(test_cmds $@) - tests+=('build_bench') # Note: Capturing strips last newline. The echo re-adds it. local num @@ -275,6 +274,7 @@ function bench { return fi echo_header "bench all" + build_bench find . -type d -iname bench-out | xargs rm -rf bench_cmds | STRICT_SCHEDULING=1 parallelise rm -rf bench-out