feat: benchmark avm simulator

.github/workflows/ci3.yml

@@ -144,12 +144,12 @@ jobs:
           max-items-in-chart: 50
 
       - name: Store yarn project benchmark result
-        if: matrix.settings.arch == 'amd64' && github.event_name == 'push' && github.ref_name == 'master' && env.SKIP_YP_BENCH != 'true'
+        if: matrix.settings.arch == 'amd64' && github.event_name == 'push' && github.ref_name == 'db/bench' && env.SKIP_YP_BENCH != 'true'
         continue-on-error: true
         uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
         with:
           name: "End-to-end Benchmark"
-          benchmark-data-dir-path: "dev/e2e-bench"
+          benchmark-data-dir-path: "dev/avm-sim-bench"
           tool: "customSmallerIsBetter"
           output-file-path: ./bench-out/yp-bench.json
           github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
@@ -160,6 +160,23 @@ jobs:
           alert-comment-cc-users: "@philwindle @spalladino"
           max-items-in-chart: 50
 
+      #- name: Store AVM simulator benchmark result
+      #  if: matrix.settings.arch == 'amd64' && github.event_name == 'push' && github.ref_name == 'db/bench' && env.SKIP_YP_BENCH != 'true'
+      #  continue-on-error: true
+      #  uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
+      #  with:
+      #    name: "AVM Simulator Benchmark"
+      #    benchmark-data-dir-path: "dev/avm-sim-bench"
+      #    tool: "customSmallerIsBetter"
+      #    output-file-path: ./bench-out/yp-bench.json
+      #    github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+      #    auto-push: true
+      #    alert-threshold: "150%"
+      #    comment-on-alert: true
+      #    fail-on-alert: false
+      #    alert-comment-cc-users: "@dbanks12"
+      #    max-items-in-chart: 50
+
       # - name: Store p2p benchmark result
       #   if: matrix.settings.arch == 'amd64' && github.event_name == 'push' && github.ref_name == 'master'
       #   continue-on-error: true

yarn-project/end-to-end/src/bench/utils.ts

@@ -1,16 +1,12 @@
 import type { AztecNodeService } from '@aztec/aztec-node';
 import { type AztecNode, BatchCall, INITIAL_L2_BLOCK_NUM, type SentTx, type WaitOpts } from '@aztec/aztec.js';
 import { mean, stdDev, times } from '@aztec/foundation/collection';
-import { randomInt } from '@aztec/foundation/crypto';
 import { BenchmarkingContract } from '@aztec/noir-contracts.js/Benchmarking';
 import { type PXEService, type PXEServiceConfig, createPXEService } from '@aztec/pxe/server';
 import type { MetricsType } from '@aztec/telemetry-client';
 import type { BenchmarkDataPoint, BenchmarkMetricsType, BenchmarkTelemetryClient } from '@aztec/telemetry-client/bench';
 
 import { writeFileSync } from 'fs';
-import { mkdirpSync } from 'fs-extra';
-import { globSync } from 'glob';
-import { join } from 'path';
 
 import { type EndToEndContext, type SetupOptions, setup } from '../fixtures/utils.js';
 
@@ -94,30 +90,6 @@ function getMetricValues(points: BenchmarkDataPoint[]) {
   }
 }
 
-/**
- * Creates and returns a directory with the current job name and a random number.
- * @param index - Index to merge into the dir path.
- * @returns A path to a created dir.
- */
-export function makeDataDirectory(index: number) {
-  const testName = expect.getState().currentTestName!.split(' ')[0].replaceAll('/', '_');
-  const db = join('data', testName, index.toString(), `${randomInt(99)}`);
-  mkdirpSync(db);
-  return db;
-}
-
-/**
- * Returns the size in disk of a folder.
- * @param path - Path to the folder.
- * @returns Size in bytes.
- */
-export function getFolderSize(path: string): number {
-  return globSync('**', { stat: true, cwd: path, nodir: true, withFileTypes: true }).reduce(
-    (accum, file) => accum + (file as any as { /** Size */ size: number }).size,
-    0,
-  );
-}
-
 /**
  * Returns a call to the benchmark contract. Each call has a private execution (account entrypoint),
  * a nested private call (create_note), a public call (increment_balance), and a nested public

yarn-project/simulator/package.json

@@ -81,12 +81,15 @@
     "@aztec/merkle-tree": "workspace:^",
     "@aztec/noir-contracts.js": "workspace:^",
     "@jest/globals": "^29.5.0",
+    "@types/fs-extra": "^11.0.2",
     "@types/jest": "^29.5.0",
     "@types/levelup": "^5.1.3",
     "@types/lodash.clonedeep": "^4.5.7",
     "@types/lodash.merge": "^4.6.9",
     "@types/memdown": "^3.0.2",
     "@types/node": "^18.7.23",
+    "fs-extra": "11.2.0",
+    "glob": "10.3.10",
     "jest": "^29.5.0",
     "jest-mock-extended": "^3.0.4",
     "ts-node": "^10.9.1",

yarn-project/simulator/src/public/avm/avm_contract_call_result.ts

@@ -17,10 +17,13 @@ export class AvmContractCallResult {
     public output: Fr[],
     public gasLeft: AvmGas,
     public revertReason?: AvmRevertReason,
+    public totalInstructions: number = 0, // including nested calls
   ) {}
 
   toString(): string {
-    let resultsStr = `reverted: ${this.reverted}, output: ${this.output}, gasLeft: ${inspect(this.gasLeft)}`;
+    let resultsStr = `reverted: ${this.reverted}, output: ${this.output}, gasLeft: ${inspect(
+      this.gasLeft,
+    )}, totalInstructions: ${this.totalInstructions}`;
     if (this.revertReason) {
       resultsStr += `, revertReason: ${this.revertReason}`;
     }
@@ -29,7 +32,13 @@ export class AvmContractCallResult {
 
   finalize(): AvmFinalizedCallResult {
     const revertReason = this.revertReason ? createSimulationError(this.revertReason, this.output) : undefined;
-    return new AvmFinalizedCallResult(this.reverted, this.output, Gas.from(this.gasLeft), revertReason);
+    return new AvmFinalizedCallResult(
+      this.reverted,
+      this.output,
+      Gas.from(this.gasLeft),
+      revertReason,
+      this.totalInstructions,
+    );
   }
 }
 
@@ -43,10 +52,13 @@ export class AvmFinalizedCallResult {
     public output: Fr[],
     public gasLeft: Gas,
     public revertReason?: SimulationError,
+    public totalInstructions: number = 0, // including nested calls
   ) {}
 
   toString(): string {
-    let resultsStr = `reverted: ${this.reverted}, output: ${this.output}, gasLeft: ${inspect(this.gasLeft)}`;
+    let resultsStr = `reverted: ${this.reverted}, output: ${this.output}, gasLeft: ${inspect(
+      this.gasLeft,
+    )}, totalInstructions: ${this.totalInstructions}`;
     if (this.revertReason) {
       resultsStr += `, revertReason: ${this.revertReason}`;
     }

yarn-project/simulator/src/public/avm/avm_machine_state.ts

@@ -38,6 +38,8 @@ export class AvmMachineState {
   public pc: number = 0;
   /** program counter of the next instruction, byte based */
   public nextPc: number = 0;
+  /** instruction counter, including nested calls */
+  public instrCounter: number = 0;
   /** return/revertdata of the last nested call. */
   public nestedReturndata: Fr[] = [];
   /** Tracks whether the last external call was successful */

yarn-project/simulator/src/public/avm/avm_simulator.ts

@@ -46,8 +46,8 @@ export class AvmSimulator {
   // only. Otherwise, use build() below.
   constructor(
     private context: AvmContext,
-    private instructionSet: InstructionSet = INSTRUCTION_SET,
     enableTallying = false,
+    private instructionSet: InstructionSet = INSTRUCTION_SET,
   ) {
     // This will be used by the CALL opcode to create a new simulator. It is required to
     // avoid a dependency cycle.
@@ -66,8 +66,8 @@ export class AvmSimulator {
 
   // Factory to have a proper function name in the logger. Retrieving the name is asynchronous and
   // cannot be done as part of the constructor.
-  public static async build(context: AvmContext): Promise<AvmSimulator> {
-    const simulator = new AvmSimulator(context);
+  public static async build(context: AvmContext, enableTallying: boolean = false): Promise<AvmSimulator> {
+    const simulator = new AvmSimulator(context, enableTallying);
     const fnName = await context.persistableState.getPublicFunctionDebugName(context.environment);
     simulator.log = createLogger(`simulator:avm(f:${fnName})`);
 
@@ -83,6 +83,7 @@ export class AvmSimulator {
     isStaticCall: boolean,
     calldata: Fr[],
     allocatedGas: Gas,
+    enableTallying: boolean = true,
   ) {
     const avmExecutionEnv = new AvmExecutionEnvironment(
       address,
@@ -96,7 +97,7 @@ export class AvmSimulator {
 
     const avmMachineState = new AvmMachineState(allocatedGas);
     const avmContext = new AvmContext(stateManager, avmExecutionEnv, avmMachineState);
-    return await AvmSimulator.build(avmContext);
+    return await AvmSimulator.build(avmContext, enableTallying);
   }
 
   /**
@@ -147,7 +148,6 @@ export class AvmSimulator {
     try {
       // Execute instruction pointed to by the current program counter
       // continuing until the machine state signifies a halt
-      let instrCounter = 0;
       while (!machineState.getHalted()) {
         // Get the instruction from cache, or deserialize for the first time
         let cachedInstruction = this.deserializedInstructionsCache.get(machineState.pc);
@@ -163,13 +163,11 @@ export class AvmSimulator {
         if (this.log.isLevelEnabled('trace')) {
           // Skip this entirely to avoid toStringing etc if trace is not enabled
           this.log.trace(
-            `[PC:${machineState.pc}] [IC:${instrCounter}] ${instruction.toString()} (gasLeft l2=${
+            `[PC:${machineState.pc}] [IC:${machineState.instrCounter}] ${instruction.toString()} (gasLeft l2=${
               machineState.l2GasLeft
             } da=${machineState.daGasLeft})`,
           );
         }
-        instrCounter++;
-
         machineState.nextPc = machineState.pc + bytesRead;
 
         // Execute the instruction.
@@ -181,6 +179,8 @@ export class AvmSimulator {
           machineState.pc += bytesRead;
         }
 
+        machineState.instrCounter++;
+
         // gas used by this instruction - used for profiling/tallying
         const gasUsed: Gas = {
           l2Gas: instrStartGas.l2Gas - machineState.l2GasLeft,
@@ -197,13 +197,19 @@ export class AvmSimulator {
       const output = machineState.getOutput();
       const reverted = machineState.getReverted();
       const revertReason = reverted ? await revertReasonFromExplicitRevert(output, this.context) : undefined;
-      const results = new AvmContractCallResult(reverted, output, machineState.gasLeft, revertReason);
+      const results = new AvmContractCallResult(
+        reverted,
+        output,
+        machineState.gasLeft,
+        revertReason,
+        machineState.instrCounter,
+      );
       this.log.debug(`Context execution results: ${results.toString()}`);
       const totalGasUsed: Gas = {
         l2Gas: callStartGas.l2Gas - machineState.l2GasLeft,
         daGas: callStartGas.daGas - machineState.daGasLeft,
       };
-      this.log.debug(`Executed ${instrCounter} instructions and consumed ${totalGasUsed.l2Gas} L2 Gas`);
+      this.log.debug(`Executed ${machineState.instrCounter} instructions and consumed ${totalGasUsed.l2Gas} L2 Gas`);
 
       this.tallyPrintFunction();
 
@@ -233,7 +239,13 @@ export class AvmSimulator {
       // Exceptional halts consume all allocated gas
       const noGasLeft = { l2Gas: 0, daGas: 0 };
       // Note: "exceptional halts" cannot return data, hence [].
-      const results = new AvmContractCallResult(/*reverted=*/ true, /*output=*/ [], noGasLeft, revertReason);
+      const results = new AvmContractCallResult(
+        /*reverted=*/ true,
+        /*output=*/ [],
+        noGasLeft,
+        revertReason,
+        machineState.instrCounter,
+      );
       this.log.debug(`Context execution results: ${results.toString()}`);
 
       this.tallyPrintFunction();

yarn-project/simulator/src/public/avm/opcodes/external_calls.ts

@@ -70,6 +70,9 @@ abstract class ExternalCall extends Instruction {
     // Track the success status directly
     context.machineState.nestedCallSuccess = success;
 
+    // Account for all instructions executed in the nested call
+    context.machineState.instrCounter += nestedCallResults.totalInstructions;
+
     // If the nested call reverted, we try to save the reason and the revert data.
     // This will be used by the caller to try to reconstruct the call stack.
     // This is only a heuristic and may not always work. It is intended to work

yarn-project/simulator/src/public/executor_metrics.ts

@@ -1,3 +1,4 @@
+import { createLogger } from '@aztec/foundation/log';
 import {
   Attributes,
   type Histogram,
@@ -13,6 +14,8 @@ export class ExecutorMetrics {
   private fnCount: UpDownCounter;
   private fnDuration: Histogram;
   private manaPerSecond: Histogram;
+  private manaUsed: UpDownCounter;
+  private totalInstructions: UpDownCounter;
   private privateEffectsInsertions: Histogram;
 
   constructor(client: TelemetryClient, name = 'PublicExecutor') {
@@ -35,18 +38,43 @@ export class ExecutorMetrics {
       valueType: ValueType.INT,
     });
 
+    this.manaUsed = meter.createUpDownCounter(Metrics.PUBLIC_EXECUTOR_SIMULATION_MANA_USED, {
+      description: 'Total mana used',
+      unit: 'mana',
+      valueType: ValueType.INT,
+    });
+
+    this.totalInstructions = meter.createUpDownCounter(Metrics.PUBLIC_EXECUTOR_SIMULATION_TOTAL_INSTRUCTIONS, {
+      description: 'Total number of instructions executed',
+      unit: 'instructions',
+      valueType: ValueType.INT,
+    });
+
     this.privateEffectsInsertions = meter.createHistogram(Metrics.PUBLIC_EXECUTION_PRIVATE_EFFECTS_INSERTION, {
       description: 'Private effects insertion time',
       unit: 'us',
       valueType: ValueType.INT,
     });
   }
 
-  recordFunctionSimulation(durationMs: number, manaUsed: number, fnName: string) {
+  recordFunctionSimulation(durationMs: number, manaUsed: number, fnName: string, totalInstructions: number) {
+    const logger = createLogger('executor-metrics');
+    logger.error(`Recording function simulation: ${fnName}`, {
+      durationMs,
+      manaUsed,
+      totalInstructions,
+    });
     this.fnCount.add(1, {
       [Attributes.OK]: true,
       [Attributes.APP_CIRCUIT_NAME]: fnName,
       [Attributes.MANA_USED]: manaUsed,
+      [Attributes.TOTAL_INSTRUCTIONS]: totalInstructions,
+    });
+    this.manaUsed.add(manaUsed, {
+      [Attributes.APP_CIRCUIT_NAME]: fnName,
+    });
+    this.totalInstructions.add(totalInstructions, {
+      [Attributes.APP_CIRCUIT_NAME]: fnName,
     });
     this.fnDuration.record(Math.ceil(durationMs));
     if (durationMs > 0 && manaUsed > 0) {

yarn-project/simulator/src/public/fixtures/public_tx_simulation_tester.ts

@@ -6,6 +6,7 @@ import { GasFees } from '@aztec/stdlib/gas';
 import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
 import { PublicCallRequest } from '@aztec/stdlib/kernel';
 import { GlobalVariables, PublicCallRequestWithCalldata, type Tx } from '@aztec/stdlib/tx';
+import type { TelemetryClient } from '@aztec/telemetry-client';
 import { NativeWorldStateService } from '@aztec/world-state';
 
 import { BaseAvmSimulationTester } from '../avm/fixtures/base_avm_simulation_tester.js';
@@ -35,14 +36,18 @@ export type TestEnqueuedCall = {
 export class PublicTxSimulationTester extends BaseAvmSimulationTester {
   private txCount = 0;
 
-  constructor(private merkleTree: MerkleTreeWriteOperations, contractDataSource: SimpleContractDataSource) {
+  constructor(
+    private merkleTree: MerkleTreeWriteOperations,
+    contractDataSource: SimpleContractDataSource,
+    private telemetryClient?: TelemetryClient,
+  ) {
     super(contractDataSource, merkleTree);
   }
 
-  public static async create(): Promise<PublicTxSimulationTester> {
+  public static async create(telemetryClient?: TelemetryClient): Promise<PublicTxSimulationTester> {
     const contractDataSource = new SimpleContractDataSource();
     const merkleTree = await (await NativeWorldStateService.tmp()).fork();
-    return new PublicTxSimulationTester(merkleTree, contractDataSource);
+    return new PublicTxSimulationTester(merkleTree, contractDataSource, telemetryClient);
   }
 
   public async createTx(
@@ -72,17 +77,26 @@ export class PublicTxSimulationTester extends BaseAvmSimulationTester {
     /* need some unique first nullifier for note-nonce computations */
     firstNullifier = new Fr(420000 + this.txCount++),
     globals = defaultGlobals(),
+    metricsTag?: string,
   ): Promise<PublicTxResult> {
     const tx = await this.createTx(sender, setupCalls, appCalls, teardownCall, feePayer, firstNullifier);
 
     await this.setFeePayerBalance(feePayer);
 
     const treesDB = new PublicTreesDB(this.merkleTree);
     const contractsDB = new PublicContractsDB(this.contractDataSource);
-    const simulator = new PublicTxSimulator(treesDB, contractsDB, globals, /*doMerkleOperations=*/ true);
+    const simulator = new PublicTxSimulator(
+      treesDB,
+      contractsDB,
+      globals,
+      /*doMerkleOperations=*/ true,
+      /*skipFeeEnforcement=*/ false,
+      this.telemetryClient,
+      /*enableCoreSimulationMetrics=*/ true,
+    );
 
     const startTime = performance.now();
-    const avmResult = await simulator.simulate(tx);
+    const avmResult = await simulator.simulate(tx, metricsTag);
     const endTime = performance.now();
     this.logger.debug(`Public transaction simulation took ${endTime - startTime}ms`);