From bf362b8572f57bdefed84ac62d375a0796833ad0 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 21 Nov 2024 09:28:44 +0000 Subject: [PATCH 01/21] First set of stats --- .../simulator/src/avm/avm_simulator.ts | 54 ++++++++++++++++++- .../src/public/public_tx_simulator.ts | 18 ++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/yarn-project/simulator/src/avm/avm_simulator.ts b/yarn-project/simulator/src/avm/avm_simulator.ts index 37c5e82feeb3..ca348caa8804 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.ts @@ -6,6 +6,7 @@ import { MAX_L2_GAS_PER_ENQUEUED_CALL, } from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; +import { Timer } from '@aztec/foundation/timer'; import { strict as assert } from 'assert'; @@ -25,6 +26,7 @@ import { } from './errors.js'; import { type AvmPersistableStateManager } from './journal/journal.js'; import { decodeInstructionFromBytecode } from './serialization/bytecode_serialization.js'; +import { Opcode } from './serialization/instruction_serialization.js'; type OpcodeTally = { count: number; @@ -81,8 +83,9 @@ export class AvmSimulator { * Fetch the bytecode and execute it in the current context. */ public async execute(): Promise { + const timer = new Timer(); const bytecode = await this.context.persistableState.getBytecode(this.context.environment.address); - + this.log.info(`Retrieved bytecode in ${timer.ms()}ms`); // This assumes that we will not be able to send messages to accounts without code // Pending classes and instances impl details if (!bytecode) { @@ -108,6 +111,12 @@ export class AvmSimulator { assert(bytecode.length > 0, "AVM simulator can't execute empty bytecode"); this.bytecode = bytecode; + let totalExecutionTime = 0; + let totalDecodeTime = 0; + + const executionTimes: number[] = []; + const decodeTimes: number[] = []; + const counts: number[] = []; const { machineState } = this.context; try { @@ -115,11 +124,25 @@ export class AvmSimulator { // continuing until the machine state signifies a halt let instrCounter = 0; while (!machineState.getHalted()) { + let timer = new Timer(); const [instruction, bytesRead] = decodeInstructionFromBytecode(bytecode, machineState.pc); assert( !!instruction, 'AVM attempted to execute non-existent instruction. This should never happen (invalid bytecode or AVM simulator bug)!', ); + let duration = timer.ms(); + totalDecodeTime += duration; + if (decodeTimes[instruction.opcode] == undefined) { + decodeTimes[instruction.opcode] = duration; + } else { + decodeTimes[instruction.opcode] += duration; + } + + if (counts[instruction.opcode] == undefined) { + counts[instruction.opcode] = 1; + } else { + counts[instruction.opcode]++; + } const instrStartGas = machineState.gasLeft; // Save gas before executing instruction (for profiling) const instrPc = machineState.pc; // Save PC before executing instruction (for profiling) @@ -133,11 +156,20 @@ export class AvmSimulator { // Normal returns and reverts will return normally here. // "Exceptional halts" will throw. machineState.nextPc = machineState.pc + bytesRead; + timer = new Timer(); await instruction.execute(this.context); if (!instruction.handlesPC()) { // Increment PC if the instruction doesn't handle it itself machineState.pc += bytesRead; } + duration = timer.ms(); + totalExecutionTime += duration; + + if (executionTimes[instruction.opcode] == undefined) { + executionTimes[instruction.opcode] = duration; + } else { + executionTimes[instruction.opcode] += duration; + } // gas used by this instruction - used for profiling/tallying const gasUsed: Gas = { @@ -158,6 +190,26 @@ export class AvmSimulator { const results = new AvmContractCallResult(reverted, output, machineState.gasLeft, revertReason); this.log.debug(`Context execution results: ${results.toString()}`); + this.log.info( + `Total decode time ${totalDecodeTime}ms, total execution time ${totalExecutionTime}ms, num instructions ${instrCounter}`, + ); + + for (let i = 0; i < decodeTimes.length; i++) { + if (decodeTimes[i] == undefined) { + continue; + } + this.log.info(`Decode Opcode ${Opcode[i]}: ${decodeTimes[i]} / ${counts[i]} = ${decodeTimes[i] / counts[i]}`); + } + + for (let i = 0; i < executionTimes.length; i++) { + if (executionTimes[i] == undefined) { + continue; + } + this.log.info( + `Execution Opcode ${Opcode[i]}: ${executionTimes[i]} / ${counts[i]} = ${executionTimes[i] / counts[i]}`, + ); + } + this.printOpcodeTallies(); // Return results for processing by calling context return results; diff --git a/yarn-project/simulator/src/public/public_tx_simulator.ts b/yarn-project/simulator/src/public/public_tx_simulator.ts index a4d312965005..9fdd8e7babf8 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.ts @@ -71,8 +71,14 @@ export class PublicTxSimulator { async simulate(tx: Tx): Promise { this.log.verbose(`Processing tx ${tx.getTxHash()}`); + let timer = new Timer(); + const context = await PublicTxContext.create(this.db, this.worldStateDB, tx, this.globalVariables); + this.log.info(`Created context in ${timer.ms()}ms`); + + timer = new Timer(); + // add new contracts to the contracts db so that their functions may be found and called // TODO(#4073): This is catching only private deployments, when we add public ones, we'll // have to capture contracts emitted in that phase as well. @@ -86,14 +92,20 @@ export class PublicTxSimulator { if (context.hasPhase(TxExecutionPhase.SETUP)) { const setupResult: ProcessedPhase = await this.simulateSetupPhase(context); processedPhases.push(setupResult); + this.log.info(`Simulated setup in ${timer.ms()}ms`); + timer = new Timer(); } if (context.hasPhase(TxExecutionPhase.APP_LOGIC)) { const appLogicResult: ProcessedPhase = await this.simulateAppLogicPhase(context); processedPhases.push(appLogicResult); + this.log.info(`Simulated app logic in ${timer.ms()}ms`); + timer = new Timer(); } if (context.hasPhase(TxExecutionPhase.TEARDOWN)) { const teardownResult: ProcessedPhase = await this.simulateTeardownPhase(context); processedPhases.push(teardownResult); + this.log.info(`Simulated teardown in ${timer.ms()}ms`); + timer = new Timer(); } context.halt(); @@ -256,6 +268,8 @@ export class PublicTxSimulator { /*l2Gas=*/ Math.min(availableGas.l2Gas, MAX_L2_GAS_PER_ENQUEUED_CALL), ); + const timer = new Timer(); + const result = await this.simulateEnqueuedCallInternal( context.state.getActiveStateManager(), executionRequest, @@ -267,7 +281,9 @@ export class PublicTxSimulator { const gasUsed = allocatedGas.sub(result.gasLeft); context.consumeGas(phase, gasUsed); this.log.verbose( - `[AVM] Enqueued public call consumed ${gasUsed.l2Gas} L2 gas ending with ${result.gasLeft.l2Gas} L2 gas left.`, + `[AVM] Enqueued public call consumed ${gasUsed.l2Gas} L2 gas ending with ${ + result.gasLeft.l2Gas + } L2 gas left. Timer ${timer.ms()}ms`, ); // TODO(dbanks12): remove once AVM proves entire public tx From 759143b28616c81dc82f8f9c8b3f4dc0d0094623 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 21 Nov 2024 17:41:53 +0000 Subject: [PATCH 02/21] WIP --- yarn-project/simulator/src/avm/avm_simulator.ts | 13 +++++++++---- .../src/avm/serialization/buffer_cursor.ts | 9 +++++++++ .../src/avm/serialization/bytecode_serialization.ts | 2 +- .../avm/serialization/instruction_serialization.ts | 12 ++++++------ .../simulator/src/public/public_processor.ts | 9 ++++++++- .../src/public/public_tx_simulator.test.ts | 3 +++ .../simulator/src/public/public_tx_simulator.ts | 5 +++++ 7 files changed, 41 insertions(+), 12 deletions(-) diff --git a/yarn-project/simulator/src/avm/avm_simulator.ts b/yarn-project/simulator/src/avm/avm_simulator.ts index ca348caa8804..442c3c293ef3 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.ts @@ -25,7 +25,11 @@ import { revertReasonFromExplicitRevert, } from './errors.js'; import { type AvmPersistableStateManager } from './journal/journal.js'; -import { decodeInstructionFromBytecode } from './serialization/bytecode_serialization.js'; +import { + INSTRUCTION_SET, + type InstructionSet, + decodeInstructionFromBytecode, +} from './serialization/bytecode_serialization.js'; import { Opcode } from './serialization/instruction_serialization.js'; type OpcodeTally = { @@ -44,7 +48,7 @@ export class AvmSimulator { private opcodeTallies: Map = new Map(); private pcTallies: Map = new Map(); - constructor(private context: AvmContext) { + constructor(private context: AvmContext, private instructionSet: InstructionSet = INSTRUCTION_SET()) { assert( context.machineState.gasLeft.l2Gas <= MAX_L2_GAS_PER_ENQUEUED_CALL, `Cannot allocate more than ${MAX_L2_GAS_PER_ENQUEUED_CALL} to the AVM for execution of an enqueued call`, @@ -62,6 +66,7 @@ export class AvmSimulator { isStaticCall: boolean, calldata: Fr[], allocatedGas: Gas, + instructionSet: InstructionSet, ) { const avmExecutionEnv = new AvmExecutionEnvironment( address, @@ -76,7 +81,7 @@ export class AvmSimulator { const avmMachineState = new AvmMachineState(allocatedGas); const avmContext = new AvmContext(stateManager, avmExecutionEnv, avmMachineState); - return new AvmSimulator(avmContext); + return new AvmSimulator(avmContext, instructionSet); } /** @@ -125,7 +130,7 @@ export class AvmSimulator { let instrCounter = 0; while (!machineState.getHalted()) { let timer = new Timer(); - const [instruction, bytesRead] = decodeInstructionFromBytecode(bytecode, machineState.pc); + const [instruction, bytesRead] = decodeInstructionFromBytecode(bytecode, machineState.pc, this.instructionSet); assert( !!instruction, 'AVM attempted to execute non-existent instruction. This should never happen (invalid bytecode or AVM simulator bug)!', diff --git a/yarn-project/simulator/src/avm/serialization/buffer_cursor.ts b/yarn-project/simulator/src/avm/serialization/buffer_cursor.ts index 237d9ae24f1e..b4d7ccd76e6f 100644 --- a/yarn-project/simulator/src/avm/serialization/buffer_cursor.ts +++ b/yarn-project/simulator/src/avm/serialization/buffer_cursor.ts @@ -10,6 +10,10 @@ export class BufferCursor { return this._position; } + public buffer(): Buffer { + return this._buffer; + } + public eof(): boolean { return this._position === this._buffer.length; } @@ -23,6 +27,11 @@ export class BufferCursor { assert(n < this._buffer.length); } + public peakUint8(): number { + const ret = this._buffer.readUint8(this._position); + return ret; + } + public readUint8(): number { const ret = this._buffer.readUint8(this._position); this._position += 1; diff --git a/yarn-project/simulator/src/avm/serialization/bytecode_serialization.ts b/yarn-project/simulator/src/avm/serialization/bytecode_serialization.ts index cebf4a73cfea..27b603664d82 100644 --- a/yarn-project/simulator/src/avm/serialization/bytecode_serialization.ts +++ b/yarn-project/simulator/src/avm/serialization/bytecode_serialization.ts @@ -63,7 +63,7 @@ export interface Deserializable { export type InstructionSet = Map; // TODO(4359): This is a function so that Call and StaticCall can be lazily resolved. // This is a temporary solution until we solve the dependency cycle. -const INSTRUCTION_SET = () => +export const INSTRUCTION_SET = () => new Map([ [Opcode.ADD_8, Add.as(Add.wireFormat8).deserialize], [Opcode.ADD_16, Add.as(Add.wireFormat16).deserialize], diff --git a/yarn-project/simulator/src/avm/serialization/instruction_serialization.ts b/yarn-project/simulator/src/avm/serialization/instruction_serialization.ts index c9ae7a3f2d29..20c25b2c9994 100644 --- a/yarn-project/simulator/src/avm/serialization/instruction_serialization.ts +++ b/yarn-project/simulator/src/avm/serialization/instruction_serialization.ts @@ -103,7 +103,7 @@ type OperandNativeType = number | bigint; type OperandWriter = (value: any) => void; // Specifies how to read and write each operand type. -const OPERAND_SPEC = new Map OperandNativeType, OperandWriter]>([ +const OPERAND_SPEC = new Map OperandNativeType, OperandWriter]>([ [OperandType.UINT8, [1, Buffer.prototype.readUint8, Buffer.prototype.writeUint8]], [OperandType.UINT16, [2, Buffer.prototype.readUint16BE, Buffer.prototype.writeUint16BE]], [OperandType.UINT32, [4, Buffer.prototype.readUint32BE, Buffer.prototype.writeUint32BE]], @@ -112,12 +112,12 @@ const OPERAND_SPEC = new Map OperandNativeType, Oper [OperandType.FF, [32, readBigInt254BE, writeBigInt254BE]], ]); -function readBigInt254BE(this: Buffer): bigint { +function readBigInt254BE(this: Buffer, offset: number): bigint { const totalBytes = 32; let ret: bigint = 0n; for (let i = 0; i < totalBytes; ++i) { ret <<= 8n; - ret |= BigInt(this.readUint8(i)); + ret |= BigInt(this.readUint8(i + offset)); } return ret; } @@ -130,12 +130,12 @@ function writeBigInt254BE(this: Buffer, value: bigint): void { } } -function readBigInt128BE(this: Buffer): bigint { +function readBigInt128BE(this: Buffer, offset: number): bigint { const totalBytes = 16; let ret: bigint = 0n; for (let i = 0; i < totalBytes; ++i) { ret <<= 8n; - ret |= BigInt(this.readUint8(i)); + ret |= BigInt(this.readUint8(i + offset)); } return ret; } @@ -163,7 +163,7 @@ export function deserialize(cursor: BufferCursor | Buffer, operands: OperandType for (const op of operands) { const opType = op; const [sizeBytes, reader, _writer] = OPERAND_SPEC.get(opType)!; - argValues.push(reader.call(cursor.bufferAtPosition())); + argValues.push(reader.call(cursor.buffer(), cursor.position())); cursor.advance(sizeBytes); } diff --git a/yarn-project/simulator/src/public/public_processor.ts b/yarn-project/simulator/src/public/public_processor.ts index 7e1e35b8710b..6ae04d269aef 100644 --- a/yarn-project/simulator/src/public/public_processor.ts +++ b/yarn-project/simulator/src/public/public_processor.ts @@ -29,6 +29,7 @@ import { Timer } from '@aztec/foundation/timer'; import { ProtocolContractAddress } from '@aztec/protocol-contracts'; import { Attributes, type TelemetryClient, type Tracer, trackSpan } from '@aztec/telemetry-client'; +import { INSTRUCTION_SET } from '../avm/serialization/bytecode_serialization.js'; import { computeFeePayerBalanceLeafSlot, computeFeePayerBalanceStorageSlot } from './fee_payment.js'; import { WorldStateDB } from './public_db_sources.js'; import { PublicProcessorMetrics } from './public_processor_metrics.js'; @@ -54,7 +55,13 @@ export class PublicProcessorFactory { const historicalHeader = maybeHistoricalHeader ?? merkleTree.getInitialHeader(); const worldStateDB = new WorldStateDB(merkleTree, this.contractDataSource); - const publicTxSimulator = new PublicTxSimulator(merkleTree, worldStateDB, this.telemetryClient, globalVariables); + const publicTxSimulator = new PublicTxSimulator( + merkleTree, + worldStateDB, + this.telemetryClient, + globalVariables, + INSTRUCTION_SET(), + ); return new PublicProcessor( merkleTree, diff --git a/yarn-project/simulator/src/public/public_tx_simulator.test.ts b/yarn-project/simulator/src/public/public_tx_simulator.test.ts index 813c074cda4f..f72073bc3c6f 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.test.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.test.ts @@ -34,6 +34,7 @@ import { type MockProxy, mock } from 'jest-mock-extended'; import { AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmPersistableStateManager } from '../avm/journal/journal.js'; +import { INSTRUCTION_SET, type InstructionSet } from '../avm/serialization/bytecode_serialization.js'; import { type WorldStateDB } from './public_db_sources.js'; import { PublicTxSimulator } from './public_tx_simulator.js'; @@ -65,6 +66,7 @@ describe('public_tx_simulator', () => { allocatedGas: Gas, transactionFee: any, fnName: any, + instructionSet: InstructionSet, ) => Promise >; @@ -186,6 +188,7 @@ describe('public_tx_simulator', () => { worldStateDB, new NoopTelemetryClient(), GlobalVariables.from({ ...GlobalVariables.empty(), gasFees }), + INSTRUCTION_SET(), /*realAvmProvingRequest=*/ false, ); diff --git a/yarn-project/simulator/src/public/public_tx_simulator.ts b/yarn-project/simulator/src/public/public_tx_simulator.ts index 9fdd8e7babf8..28f99c9d9c2f 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.ts @@ -24,6 +24,7 @@ import { type TelemetryClient } from '@aztec/telemetry-client'; import { type AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmPersistableStateManager, AvmSimulator } from '../avm/index.js'; +import { type InstructionSet } from '../avm/serialization/bytecode_serialization.js'; import { getPublicFunctionDebugName } from '../common/debug_fn_name.js'; import { ExecutorMetrics } from './executor_metrics.js'; import { type WorldStateDB } from './public_db_sources.js'; @@ -57,6 +58,7 @@ export class PublicTxSimulator { private worldStateDB: WorldStateDB, client: TelemetryClient, private globalVariables: GlobalVariables, + private instructionSet: InstructionSet, private realAvmProvingRequests: boolean = true, ) { this.log = createDebugLogger(`aztec:public_tx_simulator`); @@ -276,6 +278,7 @@ export class PublicTxSimulator { allocatedGas, context.getTransactionFee(phase), fnName, + this.instructionSet, ); const gasUsed = allocatedGas.sub(result.gasLeft); @@ -327,6 +330,7 @@ export class PublicTxSimulator { allocatedGas: Gas, transactionFee: Fr, fnName: string, + instructionSet: InstructionSet, ): Promise { const address = executionRequest.callContext.contractAddress; const sender = executionRequest.callContext.msgSender; @@ -347,6 +351,7 @@ export class PublicTxSimulator { executionRequest.callContext.isStaticCall, executionRequest.args, allocatedGas, + instructionSet, ); const avmCallResult = await simulator.execute(); const result = avmCallResult.finalize(); From c52e2fd7e930049f85e59d115eca2765ad232edb Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 22 Nov 2024 18:49:52 +0000 Subject: [PATCH 03/21] WIP --- .../barretenberg/crypto/poseidon2/c_bind.cpp | 14 ++++++++++ barretenberg/ts/src/barretenberg_api/index.ts | 12 ++++++++ .../archiver/src/archiver/archiver.ts | 28 ++++++++++++++++--- .../archiver/src/archiver/archiver_store.ts | 4 ++- .../src/archiver/archiver_store_test_suite.ts | 13 +++++++-- .../kv_archiver_store/contract_class_store.ts | 15 +++++++++- .../kv_archiver_store/kv_archiver_store.ts | 23 +++++++++++++-- .../memory_archiver_store.ts | 19 +++++++++++-- yarn-project/archiver/src/factory.ts | 12 ++++++-- .../src/interfaces/archiver.test.ts | 6 ++++ .../circuit-types/src/interfaces/archiver.ts | 1 + .../src/contract/contract_class_id.ts | 9 ++---- .../interfaces/contract_data_source.ts | 2 ++ .../foundation/src/crypto/poseidon/index.ts | 18 ++++++++++++ .../simulator/src/avm/journal/journal.ts | 24 +++++++++++++++- 15 files changed, 176 insertions(+), 24 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp index f6e612b3b38d..ef8415a340b6 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp @@ -47,3 +47,17 @@ WASM_EXPORT void poseidon2_permutation(fr::vec_in_buf inputs_buffer, fr::vec_out const std::vector results(results_array.begin(), results_array.end()); *output = to_heap_buffer(results); } + +WASM_EXPORT void poseidon2_hash_accumulate(fr::vec_in_buf inputs_buffer, fr::out_buf output) +{ + std::vector to_hash; + read(inputs_buffer, to_hash); + const size_t numHashes = to_hash.size(); + fr result = 0; + size_t count = 0; + while (count < numHashes) { + result = crypto::Poseidon2::hash({ to_hash[count], result }); + ++count; + } + write(output, result); +} diff --git a/barretenberg/ts/src/barretenberg_api/index.ts b/barretenberg/ts/src/barretenberg_api/index.ts index 202239039dce..02cfcf5bf100 100644 --- a/barretenberg/ts/src/barretenberg_api/index.ts +++ b/barretenberg/ts/src/barretenberg_api/index.ts @@ -704,6 +704,18 @@ export class BarretenbergApiSync { return out[0]; } + poseidon2HashAccumulate(inputsBuffer: Fr[]): Fr { + const inArgs = [inputsBuffer].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = this.wasm.callWasmExport( + 'poseidon2_hash_accumulate', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + poseidon2Hashes(inputsBuffer: Fr[]): Fr { const inArgs = [inputsBuffer].map(serializeBufferable); const outTypes: OutputType[] = [Fr]; diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index cd1c78cb3b27..72a1902bdb85 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -33,6 +33,7 @@ import { type PublicFunction, UnconstrainedFunctionBroadcastedEvent, type UnconstrainedFunctionWithMembershipProof, + computePublicBytecodeCommitment, isValidPrivateFunctionMembershipProof, isValidUnconstrainedFunctionMembershipProof, } from '@aztec/circuits.js'; @@ -703,6 +704,10 @@ export class Archiver implements ArchiveSource { return this.store.getContractClass(id); } + public getBytecodeCommitment(id: Fr): Promise { + return this.store.getBytecodeCommitment(id); + } + public getContract(address: AztecAddress): Promise { return this.store.getContractInstance(address); } @@ -731,7 +736,11 @@ export class Archiver implements ArchiveSource { // TODO(#10007): Remove this method async addContractClass(contractClass: ContractClassPublic): Promise { - await this.store.addContractClasses([contractClass], 0); + await this.store.addContractClasses( + [contractClass], + [computePublicBytecodeCommitment(contractClass.packedBytecode)], + 0, + ); return; } @@ -801,8 +810,12 @@ class ArchiverStoreHelper constructor(private readonly store: ArchiverDataStore) {} // TODO(#10007): Remove this method - addContractClasses(contractClasses: ContractClassPublic[], blockNum: number): Promise { - return this.store.addContractClasses(contractClasses, blockNum); + addContractClasses( + contractClasses: ContractClassPublic[], + bytecodeCommitments: Fr[], + blockNum: number, + ): Promise { + return this.store.addContractClasses(contractClasses, bytecodeCommitments, blockNum); } /** @@ -817,7 +830,11 @@ class ArchiverStoreHelper if (contractClasses.length > 0) { contractClasses.forEach(c => this.#log.verbose(`Registering contract class ${c.id.toString()}`)); if (operation == Operation.Store) { - return await this.store.addContractClasses(contractClasses, blockNum); + return await this.store.addContractClasses( + contractClasses, + contractClasses.map(x => computePublicBytecodeCommitment(x.packedBytecode)), + blockNum, + ); } else if (operation == Operation.Delete) { return await this.store.deleteContractClasses(contractClasses, blockNum); } @@ -1025,6 +1042,9 @@ class ArchiverStoreHelper getContractClass(id: Fr): Promise { return this.store.getContractClass(id); } + getBytecodeCommitment(contractClassId: Fr): Promise { + return this.store.getBytecodeCommitment(contractClassId); + } getContractInstance(address: AztecAddress): Promise { return this.store.getContractInstance(address); } diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 12a6d0e1f966..371ea5c2348c 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -229,10 +229,12 @@ export interface ArchiverDataStore { * @param blockNumber - Number of the L2 block the contracts were registered in. * @returns True if the operation is successful. */ - addContractClasses(data: ContractClassPublic[], blockNumber: number): Promise; + addContractClasses(data: ContractClassPublic[], bytecodeCommitments: Fr[], blockNumber: number): Promise; deleteContractClasses(data: ContractClassPublic[], blockNumber: number): Promise; + getBytecodeCommitment(contractClassId: Fr): Promise; + /** * Returns a contract class given its id, or undefined if not exists. * @param id - Id of the contract class. diff --git a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts index 9253a99fede0..b96afbd284d3 100644 --- a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts +++ b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts @@ -9,6 +9,7 @@ import { L1_TO_L2_MSG_SUBTREE_HEIGHT, MAX_NULLIFIERS_PER_TX, SerializableContractInstance, + computePublicBytecodeCommitment, } from '@aztec/circuits.js'; import { makeContractClassPublic, @@ -289,7 +290,11 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch beforeEach(async () => { contractClass = makeContractClassPublic(); - await store.addContractClasses([contractClass], blockNum); + await store.addContractClasses( + [contractClass], + [computePublicBytecodeCommitment(contractClass.packedBytecode)], + blockNum, + ); }); it('returns previously stored contract class', async () => { @@ -302,7 +307,11 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); it('returns contract class if later "deployment" class was deleted', async () => { - await store.addContractClasses([contractClass], blockNum + 1); + await store.addContractClasses( + [contractClass], + [computePublicBytecodeCommitment(contractClass.packedBytecode)], + blockNum + 1, + ); await store.deleteContractClasses([contractClass], blockNum + 1); await expect(store.getContractClass(contractClass.id)).resolves.toMatchObject(contractClass); }); diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/contract_class_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/contract_class_store.ts index 2c63b150ad20..ce8bbb823ce9 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/contract_class_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/contract_class_store.ts @@ -15,22 +15,30 @@ import { type AztecKVStore, type AztecMap } from '@aztec/kv-store'; */ export class ContractClassStore { #contractClasses: AztecMap; + #bytecodeCommitments: AztecMap; constructor(private db: AztecKVStore) { this.#contractClasses = db.openMap('archiver_contract_classes'); + this.#bytecodeCommitments = db.openMap('archiver_bytecode_commitments'); } - async addContractClass(contractClass: ContractClassPublic, blockNumber: number): Promise { + async addContractClass( + contractClass: ContractClassPublic, + bytecodeCommitment: Fr, + blockNumber: number, + ): Promise { await this.#contractClasses.setIfNotExists( contractClass.id.toString(), serializeContractClassPublic({ ...contractClass, l2BlockNumber: blockNumber }), ); + await this.#bytecodeCommitments.setIfNotExists(contractClass.id.toString(), bytecodeCommitment.toBuffer()); } async deleteContractClasses(contractClass: ContractClassPublic, blockNumber: number): Promise { const restoredContractClass = this.#contractClasses.get(contractClass.id.toString()); if (restoredContractClass && deserializeContractClassPublic(restoredContractClass).l2BlockNumber >= blockNumber) { await this.#contractClasses.delete(contractClass.id.toString()); + await this.#bytecodeCommitments.delete(contractClass.id.toString()); } } @@ -39,6 +47,11 @@ export class ContractClassStore { return contractClass && { ...deserializeContractClassPublic(contractClass), id }; } + getBytecodeCommitment(id: Fr): Fr | undefined { + const value = this.#bytecodeCommitments.get(id.toString()); + return value === undefined ? undefined : Fr.fromBuffer(value); + } + getContractClassIds(): Fr[] { return Array.from(this.#contractClasses.keys()).map(key => Fr.fromString(key)); } diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index dcbf3d3691ee..bd9c268ce427 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -22,6 +22,7 @@ import { import { type ContractArtifact } from '@aztec/foundation/abi'; import { type AztecAddress } from '@aztec/foundation/aztec-address'; import { createDebugLogger } from '@aztec/foundation/log'; +import { Timer } from '@aztec/foundation/timer'; import { type AztecKVStore } from '@aztec/kv-store'; import { type ArchiverDataStore, type ArchiverL1SynchPoint } from '../archiver_store.js'; @@ -76,11 +77,23 @@ export class KVArchiverDataStore implements ArchiverDataStore { } getContractInstance(address: AztecAddress): Promise { - return Promise.resolve(this.#contractInstanceStore.getContractInstance(address)); + const timer = new Timer(); + const contract = this.#contractInstanceStore.getContractInstance(address); + this.#log.info(`Retrieved contract in ${timer.ms()}ms`); + + return Promise.resolve(contract); } - async addContractClasses(data: ContractClassPublic[], blockNumber: number): Promise { - return (await Promise.all(data.map(c => this.#contractClassStore.addContractClass(c, blockNumber)))).every(Boolean); + async addContractClasses( + data: ContractClassPublic[], + bytecodeCommitments: Fr[], + blockNumber: number, + ): Promise { + return ( + await Promise.all( + data.map((c, i) => this.#contractClassStore.addContractClass(c, bytecodeCommitments[i], blockNumber)), + ) + ).every(Boolean); } async deleteContractClasses(data: ContractClassPublic[], blockNumber: number): Promise { @@ -89,6 +102,10 @@ export class KVArchiverDataStore implements ArchiverDataStore { ); } + getBytecodeCommitment(contractClassId: Fr): Promise { + return Promise.resolve(this.#contractClassStore.getBytecodeCommitment(contractClassId)); + } + addFunctions( contractClassId: Fr, privateFunctions: ExecutablePrivateFunctionWithMembershipProof[], diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index 7f5418f79a6a..62f8f3b301d2 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -78,6 +78,8 @@ export class MemoryArchiverStore implements ArchiverDataStore { private contractClasses: Map = new Map(); + private bytecodeCommitments: Map = new Map(); + private privateFunctions: Map = new Map(); private unconstrainedFunctions: Map = new Map(); @@ -116,6 +118,10 @@ export class MemoryArchiverStore implements ArchiverDataStore { return Promise.resolve(this.contractInstances.get(address.toString())); } + public getBytecodeCommitment(contractClassId: Fr): Promise { + return Promise.resolve(this.bytecodeCommitments.get(contractClassId.toString())); + } + public addFunctions( contractClassId: Fr, newPrivateFunctions: ExecutablePrivateFunctionWithMembershipProof[], @@ -138,14 +144,22 @@ export class MemoryArchiverStore implements ArchiverDataStore { return Promise.resolve(true); } - public addContractClasses(data: ContractClassPublic[], blockNumber: number): Promise { - for (const contractClass of data) { + public addContractClasses( + data: ContractClassPublic[], + bytecodeCommitments: Fr[], + blockNumber: number, + ): Promise { + for (let i = 0; i < data.length; i++) { + const contractClass = data[i]; if (!this.contractClasses.has(contractClass.id.toString())) { this.contractClasses.set(contractClass.id.toString(), { ...contractClass, l2BlockNumber: blockNumber, }); } + if (!this.bytecodeCommitments.has(contractClass.id.toString())) { + this.bytecodeCommitments.set(contractClass.id.toString(), bytecodeCommitments[i]); + } } return Promise.resolve(true); } @@ -155,6 +169,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { const restored = this.contractClasses.get(contractClass.id.toString()); if (restored && restored.l2BlockNumber >= blockNumber) { this.contractClasses.delete(contractClass.id.toString()); + this.bytecodeCommitments.delete(contractClass.id.toString()); } } return Promise.resolve(true); diff --git a/yarn-project/archiver/src/factory.ts b/yarn-project/archiver/src/factory.ts index 28ed2ec035bb..a2bd3c66ac7a 100644 --- a/yarn-project/archiver/src/factory.ts +++ b/yarn-project/archiver/src/factory.ts @@ -1,5 +1,9 @@ import { type ArchiverApi, type Service } from '@aztec/circuit-types'; -import { type ContractClassPublic, getContractClassFromArtifact } from '@aztec/circuits.js'; +import { + type ContractClassPublic, + computePublicBytecodeCommitment, + getContractClassFromArtifact, +} from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { type Maybe } from '@aztec/foundation/types'; import { type DataStoreConfig } from '@aztec/kv-store/config'; @@ -40,7 +44,8 @@ async function registerProtocolContracts(store: KVArchiverDataStore) { unconstrainedFunctions: [], }; await store.addContractArtifact(contract.address, contract.artifact); - await store.addContractClasses([contractClassPublic], blockNumber); + const bytecodeCommitment = computePublicBytecodeCommitment(contractClassPublic.packedBytecode); + await store.addContractClasses([contractClassPublic], [bytecodeCommitment], blockNumber); await store.addContractInstances([contract.instance], blockNumber); } } @@ -58,5 +63,6 @@ async function registerCommonContracts(store: KVArchiverDataStore) { privateFunctions: [], unconstrainedFunctions: [], })); - await store.addContractClasses(classes, blockNumber); + const bytecodeCommitments = classes.map(x => computePublicBytecodeCommitment(x.packedBytecode)); + await store.addContractClasses(classes, bytecodeCommitments, blockNumber); } diff --git a/yarn-project/circuit-types/src/interfaces/archiver.test.ts b/yarn-project/circuit-types/src/interfaces/archiver.test.ts index 6e9dacdd574d..3c5385bb5e57 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.test.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.test.ts @@ -8,6 +8,7 @@ import { Header, type PublicFunction, PublicKeys, + computePublicBytecodeCommitment, getContractClassFromArtifact, } from '@aztec/circuits.js'; import { type ContractArtifact } from '@aztec/foundation/abi'; @@ -359,6 +360,11 @@ class MockArchiver implements ArchiverApi { const contractClass = getContractClassFromArtifact(this.artifact); return Promise.resolve({ ...contractClass, unconstrainedFunctions: [], privateFunctions: [] }); } + getBytecodeCommitment(id: Fr): Promise { + expect(id).toBeInstanceOf(Fr); + const contractClass = getContractClassFromArtifact(this.artifact); + return Promise.resolve(computePublicBytecodeCommitment(contractClass.packedBytecode)); + } getContract(address: AztecAddress): Promise { return Promise.resolve({ address, diff --git a/yarn-project/circuit-types/src/interfaces/archiver.ts b/yarn-project/circuit-types/src/interfaces/archiver.ts index f5a212f50990..10b2edd9b6a4 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.ts @@ -70,6 +70,7 @@ export const ArchiverApiSchema: ApiSchemaFor = { .args(schemas.AztecAddress, schemas.FunctionSelector) .returns(PublicFunctionSchema.optional()), getContractClass: z.function().args(schemas.Fr).returns(ContractClassPublicSchema.optional()), + getBytecodeCommitment: z.function().args(schemas.Fr).returns(schemas.Fr), getContract: z.function().args(schemas.AztecAddress).returns(ContractInstanceWithAddressSchema.optional()), getContractClassIds: z.function().args().returns(z.array(schemas.Fr)), getContractArtifact: z.function().args(schemas.AztecAddress).returns(ContractArtifactSchema.optional()), diff --git a/yarn-project/circuits.js/src/contract/contract_class_id.ts b/yarn-project/circuits.js/src/contract/contract_class_id.ts index 360a498ff618..af72181b2266 100644 --- a/yarn-project/circuits.js/src/contract/contract_class_id.ts +++ b/yarn-project/circuits.js/src/contract/contract_class_id.ts @@ -1,5 +1,5 @@ import { bufferAsFields } from '@aztec/foundation/abi'; -import { poseidon2Hash, poseidon2HashWithSeparator } from '@aztec/foundation/crypto'; +import { poseidon2Hash, poseidon2HashAccumulate, poseidon2HashWithSeparator } from '@aztec/foundation/crypto'; import { Fr } from '@aztec/foundation/fields'; import { strict as assert } from 'assert'; @@ -66,10 +66,5 @@ export function computePublicBytecodeCommitment(packedBytecode: Buffer) { const bytecodeLength = Math.ceil(encodedBytecode[0].toNumber() / (Fr.SIZE_IN_BYTES - 1)); assert(bytecodeLength < MAX_PACKED_PUBLIC_BYTECODE_SIZE_IN_FIELDS, 'Bytecode exceeds maximum deployable size'); - let bytecodeCommitment = new Fr(0); - for (let i = 0; i < bytecodeLength; i++) { - // We skip the first element, which is the length of the bytecode - bytecodeCommitment = poseidon2Hash([encodedBytecode[i + 1], bytecodeCommitment]); - } - return bytecodeCommitment; + return bytecodeLength == 0 ? new Fr(0) : poseidon2HashAccumulate(encodedBytecode.slice(1, bytecodeLength + 1)); } diff --git a/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts b/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts index b67afc8ac50e..bb0e92e40fc5 100644 --- a/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts +++ b/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts @@ -26,6 +26,8 @@ export interface ContractDataSource { */ getContractClass(id: Fr): Promise; + getBytecodeCommitment(id: Fr): Promise; + /** * Adds a contract class to the database. * TODO(#10007): Remove this method diff --git a/yarn-project/foundation/src/crypto/poseidon/index.ts b/yarn-project/foundation/src/crypto/poseidon/index.ts index 1e1274ec7815..53aee7f64b64 100644 --- a/yarn-project/foundation/src/crypto/poseidon/index.ts +++ b/yarn-project/foundation/src/crypto/poseidon/index.ts @@ -41,6 +41,24 @@ export function poseidon2HashWithSeparator(input: Fieldable[], separator: number ); } +/** + * Create a poseidon hash (field) from an array of input fields and a domain separator. + * @param input - The input fields to hash. + * @returns The poseidon hash. + */ +export function poseidon2HashAccumulate(input: Fieldable[]): Fr { + const inputFields = serializeToFields(input); + return Fr.fromBuffer( + Buffer.from( + BarretenbergSync.getSingleton() + .poseidon2HashAccumulate( + inputFields.map(i => new FrBarretenberg(i.toBuffer())), // TODO(#4189): remove this stupid conversion + ) + .toBuffer(), + ), + ); +} + /** * Runs a Poseidon2 permutation. * @param input the input state. Expected to be of size 4. diff --git a/yarn-project/simulator/src/avm/journal/journal.ts b/yarn-project/simulator/src/avm/journal/journal.ts index 637ec02d7fe7..446148ad861c 100644 --- a/yarn-project/simulator/src/avm/journal/journal.ts +++ b/yarn-project/simulator/src/avm/journal/journal.ts @@ -12,6 +12,7 @@ import { import { computePublicDataTreeLeafSlot, siloNoteHash, siloNullifier } from '@aztec/circuits.js/hash'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; +import { Timer } from '@aztec/foundation/timer'; import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import assert from 'assert'; @@ -469,24 +470,42 @@ export class AvmPersistableStateManager { */ public async getBytecode(contractAddress: AztecAddress): Promise { this.log.debug(`Getting bytecode for contract address ${contractAddress}`); + let timer = new Timer(); const instanceWithAddress = await this.worldStateDB.getContractInstance(contractAddress); const exists = instanceWithAddress !== undefined; + let duration = timer.ms(); + this.log.info(`Retrieved contract in ${duration}ms`); if (exists) { const instance = new SerializableContractInstance(instanceWithAddress); + timer = new Timer(); const contractClass = await this.worldStateDB.getContractClass(instance.contractClassId); + duration = timer.ms(); + this.log.info(`Retrieved contract class in ${duration}ms`); + + const bytecodeCommitment = await this.worldStateDB.getBytecodeCommitment(instance.contractClassId); + assert( contractClass, `Contract class not found in DB, but a contract instance was found with this class ID (${instance.contractClassId}). This should not happen!`, ); + assert( + bytecodeCommitment, + `Bytecode commitment was not found in DB for contract class (${instance.contractClassId}). This should not happen!`, + ); + + timer = new Timer(); const contractClassPreimage = { artifactHash: contractClass.artifactHash, privateFunctionsRoot: contractClass.privateFunctionsRoot, - publicBytecodeCommitment: computePublicBytecodeCommitment(contractClass.packedBytecode), + publicBytecodeCommitment: bytecodeCommitment, }; + duration = timer.ms(); + this.log.info(`Computed bytecode commitment in ${duration}ms`); + timer = new Timer(); this.trace.traceGetBytecode( contractAddress, exists, @@ -494,6 +513,9 @@ export class AvmPersistableStateManager { instance, contractClassPreimage, ); + duration = timer.ms(); + this.log.info(`Retrieved bytecode in ${duration}ms`); + return contractClass.packedBytecode; } else { // If the contract instance is not found, we assume it has not been deployed. From 3c8e324d2cb00ba36fe2beefd181043c15f9e5d9 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 22 Nov 2024 18:50:08 +0000 Subject: [PATCH 04/21] WIP --- .../simulator/src/public/public_db_sources.ts | 31 +++++++++++++++++-- .../simulator/src/public/public_processor.ts | 9 +----- .../src/public/public_tx_simulator.test.ts | 1 - .../src/public/public_tx_simulator.ts | 4 +-- .../util/txe_public_contract_data_source.ts | 6 ++++ 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/yarn-project/simulator/src/public/public_db_sources.ts b/yarn-project/simulator/src/public/public_db_sources.ts index ce9b66b0da60..ea02a4d82aaa 100644 --- a/yarn-project/simulator/src/public/public_db_sources.ts +++ b/yarn-project/simulator/src/public/public_db_sources.ts @@ -19,6 +19,7 @@ import { type NULLIFIER_TREE_HEIGHT, type NullifierLeafPreimage, type PublicDataTreeLeafPreimage, + computePublicBytecodeCommitment, } from '@aztec/circuits.js'; import { computeL1ToL2MessageNullifier, computePublicDataTreeLeafSlot } from '@aztec/circuits.js/hash'; import { createDebugLogger } from '@aztec/foundation/log'; @@ -38,11 +39,11 @@ import { export class ContractsDataSourcePublicDB implements PublicContractsDB { private instanceCache = new Map(); private classCache = new Map(); + private bytecodeCommitmentCache = new Map(); private log = createDebugLogger('aztec:sequencer:contracts-data-source'); constructor(private dataSource: ContractDataSource) {} - /** * Add new contracts from a transaction * @param tx - The transaction to add contracts from. @@ -52,7 +53,8 @@ export class ContractsDataSourcePublicDB implements PublicContractsDB { const logs = tx.contractClassLogs.unrollLogs(); ContractClassRegisteredEvent.fromLogs(logs, ProtocolContractAddress.ContractClassRegisterer).forEach(e => { this.log.debug(`Adding class ${e.contractClassId.toString()} to public execution contract cache`); - this.classCache.set(e.contractClassId.toString(), e.toContractClassPublic()); + const key = e.contractClassId.toString(); + this.classCache.set(key, e.toContractClassPublic()); }); // We store the contract instance deployed event log in enc logs, contract_instance_deployer_contract/src/main.nr const encLogs = tx.encryptedLogs.unrollLogs(); @@ -92,6 +94,31 @@ export class ContractsDataSourcePublicDB implements PublicContractsDB { return this.classCache.get(contractClassId.toString()) ?? (await this.dataSource.getContractClass(contractClassId)); } + public async getBytecodeCommitment(contractClassId: Fr): Promise { + // Try and retrieve from cache + const key = contractClassId.toString(); + const result = this.bytecodeCommitmentCache.get(key); + if (result !== undefined) { + return result; + } + // Now try from the store + const fromStore = await this.dataSource.getBytecodeCommitment(contractClassId); + if (fromStore !== undefined) { + this.bytecodeCommitmentCache.set(key, fromStore); + return fromStore; + } + + // Not in either the store or the cache, build it here and cache + const contractClass = await this.getContractClass(contractClassId); + if (contractClass === undefined) { + return undefined; + } + + const value = computePublicBytecodeCommitment(contractClass.packedBytecode); + this.bytecodeCommitmentCache.set(key, value); + return value; + } + async getBytecode(address: AztecAddress, selector: FunctionSelector): Promise { const instance = await this.getContractInstance(address); if (!instance) { diff --git a/yarn-project/simulator/src/public/public_processor.ts b/yarn-project/simulator/src/public/public_processor.ts index 6ae04d269aef..7e1e35b8710b 100644 --- a/yarn-project/simulator/src/public/public_processor.ts +++ b/yarn-project/simulator/src/public/public_processor.ts @@ -29,7 +29,6 @@ import { Timer } from '@aztec/foundation/timer'; import { ProtocolContractAddress } from '@aztec/protocol-contracts'; import { Attributes, type TelemetryClient, type Tracer, trackSpan } from '@aztec/telemetry-client'; -import { INSTRUCTION_SET } from '../avm/serialization/bytecode_serialization.js'; import { computeFeePayerBalanceLeafSlot, computeFeePayerBalanceStorageSlot } from './fee_payment.js'; import { WorldStateDB } from './public_db_sources.js'; import { PublicProcessorMetrics } from './public_processor_metrics.js'; @@ -55,13 +54,7 @@ export class PublicProcessorFactory { const historicalHeader = maybeHistoricalHeader ?? merkleTree.getInitialHeader(); const worldStateDB = new WorldStateDB(merkleTree, this.contractDataSource); - const publicTxSimulator = new PublicTxSimulator( - merkleTree, - worldStateDB, - this.telemetryClient, - globalVariables, - INSTRUCTION_SET(), - ); + const publicTxSimulator = new PublicTxSimulator(merkleTree, worldStateDB, this.telemetryClient, globalVariables); return new PublicProcessor( merkleTree, diff --git a/yarn-project/simulator/src/public/public_tx_simulator.test.ts b/yarn-project/simulator/src/public/public_tx_simulator.test.ts index f72073bc3c6f..9ffd115efeef 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.test.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.test.ts @@ -188,7 +188,6 @@ describe('public_tx_simulator', () => { worldStateDB, new NoopTelemetryClient(), GlobalVariables.from({ ...GlobalVariables.empty(), gasFees }), - INSTRUCTION_SET(), /*realAvmProvingRequest=*/ false, ); diff --git a/yarn-project/simulator/src/public/public_tx_simulator.ts b/yarn-project/simulator/src/public/public_tx_simulator.ts index 28f99c9d9c2f..847cf5bb01f5 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.ts @@ -24,7 +24,7 @@ import { type TelemetryClient } from '@aztec/telemetry-client'; import { type AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmPersistableStateManager, AvmSimulator } from '../avm/index.js'; -import { type InstructionSet } from '../avm/serialization/bytecode_serialization.js'; +import { INSTRUCTION_SET, type InstructionSet } from '../avm/serialization/bytecode_serialization.js'; import { getPublicFunctionDebugName } from '../common/debug_fn_name.js'; import { ExecutorMetrics } from './executor_metrics.js'; import { type WorldStateDB } from './public_db_sources.js'; @@ -52,13 +52,13 @@ export class PublicTxSimulator { metrics: ExecutorMetrics; private log: DebugLogger; + private instructionSet: InstructionSet = INSTRUCTION_SET(); constructor( private db: MerkleTreeReadOperations, private worldStateDB: WorldStateDB, client: TelemetryClient, private globalVariables: GlobalVariables, - private instructionSet: InstructionSet, private realAvmProvingRequests: boolean = true, ) { this.log = createDebugLogger(`aztec:public_tx_simulator`); diff --git a/yarn-project/txe/src/util/txe_public_contract_data_source.ts b/yarn-project/txe/src/util/txe_public_contract_data_source.ts index 1ae75b00a600..d38bc684f1a9 100644 --- a/yarn-project/txe/src/util/txe_public_contract_data_source.ts +++ b/yarn-project/txe/src/util/txe_public_contract_data_source.ts @@ -7,6 +7,7 @@ import { FunctionSelector, PUBLIC_DISPATCH_SELECTOR, type PublicFunction, + computePublicBytecodeCommitment, } from '@aztec/circuits.js'; import { type ContractArtifact } from '@aztec/foundation/abi'; import { PrivateFunctionsTree } from '@aztec/pxe'; @@ -54,6 +55,11 @@ export class TXEPublicContractDataSource implements ContractDataSource { }; } + async getBytecodeCommitment(id: Fr): Promise { + const contractClass = await this.txeOracle.getContractDataOracle().getContractClass(id); + return Promise.resolve(computePublicBytecodeCommitment(contractClass.packedBytecode)); + } + async getContract(address: AztecAddress): Promise { const instance = await this.txeOracle.getContractDataOracle().getContractInstance(address); return { ...instance, address }; From 7d46400fca4c92fa06d44ca7f62c8886f5228096 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sat, 23 Nov 2024 18:08:04 +0000 Subject: [PATCH 05/21] WIP --- yarn-project/archiver/src/archiver/archiver.ts | 7 +++++++ .../archiver/src/archiver/archiver_store.ts | 3 ++- .../kv_archiver_store/kv_archiver_store.ts | 14 +++++++++++--- .../memory_archiver_store.ts | 15 ++++++++++++++- .../src/interfaces/archiver.test.ts | 9 +++++++++ .../circuit-types/src/interfaces/archiver.ts | 4 ++++ .../contract/interfaces/contract_data_source.ts | 2 ++ .../simulator/src/public/public_db_sources.ts | 16 ++-------------- .../src/util/txe_public_contract_data_source.ts | 11 +++++++++++ 9 files changed, 62 insertions(+), 19 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 72a1902bdb85..53564a70c1d9 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -752,6 +752,10 @@ export class Archiver implements ArchiveSource { return this.store.getContractArtifact(address); } + getContractFunctionName(address: AztecAddress, selector: FunctionSelector): Promise { + return this.store.getContractFunctionName(address, selector); + } + async getL2Tips(): Promise { const [latestBlockNumber, provenBlockNumber] = await Promise.all([ this.getBlockNumber(), @@ -1057,6 +1061,9 @@ class ArchiverStoreHelper getContractArtifact(address: AztecAddress): Promise { return this.store.getContractArtifact(address); } + getContractFunctionName(address: AztecAddress, selector: FunctionSelector): Promise { + return this.store.getContractFunctionName(address, selector); + } getTotalL1ToL2MessageCount(): Promise { return this.store.getTotalL1ToL2MessageCount(); } diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 371ea5c2348c..61a2c9a5a28b 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -20,7 +20,7 @@ import { type Header, type UnconstrainedFunctionWithMembershipProof, } from '@aztec/circuits.js'; -import { type ContractArtifact } from '@aztec/foundation/abi'; +import { type ContractArtifact, type FunctionSelector } from '@aztec/foundation/abi'; import { type AztecAddress } from '@aztec/foundation/aztec-address'; import { type DataRetrieval } from './structs/data_retrieval.js'; @@ -270,4 +270,5 @@ export interface ArchiverDataStore { addContractArtifact(address: AztecAddress, contract: ContractArtifact): Promise; getContractArtifact(address: AztecAddress): Promise; + getContractFunctionName(address: AztecAddress, selector: FunctionSelector): Promise; } diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index bd9c268ce427..6a03c79869b3 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -19,7 +19,7 @@ import { type Header, type UnconstrainedFunctionWithMembershipProof, } from '@aztec/circuits.js'; -import { type ContractArtifact } from '@aztec/foundation/abi'; +import { type ContractArtifact, FunctionSelector } from '@aztec/foundation/abi'; import { type AztecAddress } from '@aztec/foundation/aztec-address'; import { createDebugLogger } from '@aztec/foundation/log'; import { Timer } from '@aztec/foundation/timer'; @@ -47,6 +47,7 @@ export class KVArchiverDataStore implements ArchiverDataStore { #contractClassStore: ContractClassStore; #contractInstanceStore: ContractInstanceStore; #contractArtifactStore: ContractArtifactsStore; + private functionNames = new Map(); #log = createDebugLogger('aztec:archiver:data-store'); @@ -64,8 +65,15 @@ export class KVArchiverDataStore implements ArchiverDataStore { return Promise.resolve(this.#contractArtifactStore.getContractArtifact(address)); } - addContractArtifact(address: AztecAddress, contract: ContractArtifact): Promise { - return this.#contractArtifactStore.addContractArtifact(address, contract); + getContractFunctionName(address: AztecAddress, selector: FunctionSelector): Promise { + return Promise.resolve(this.functionNames.get(selector.toString())); + } + + async addContractArtifact(address: AztecAddress, contract: ContractArtifact): Promise { + await this.#contractArtifactStore.addContractArtifact(address, contract); + contract.functions.forEach(f => { + this.functionNames.set(FunctionSelector.fromNameAndParameters(f.name, f.parameters).toString(), f.name); + }); } getContractClass(id: Fr): Promise { diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index 62f8f3b301d2..30c974553581 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -32,7 +32,7 @@ import { MAX_NULLIFIERS_PER_TX, type UnconstrainedFunctionWithMembershipProof, } from '@aztec/circuits.js'; -import { type ContractArtifact } from '@aztec/foundation/abi'; +import { type ContractArtifact, FunctionSelector } from '@aztec/foundation/abi'; import { type AztecAddress } from '@aztec/foundation/aztec-address'; import { createDebugLogger } from '@aztec/foundation/log'; @@ -751,4 +751,17 @@ export class MemoryArchiverStore implements ArchiverDataStore { public getContractArtifact(address: AztecAddress): Promise { return Promise.resolve(this.contractArtifacts.get(address.toString())); } + + async getContractFunctionName(address: AztecAddress, selector: FunctionSelector): Promise { + const artifact = await this.getContractArtifact(address); + + if (!artifact) { + return undefined; + } + + const func = artifact.functions.find(f => + FunctionSelector.fromNameAndParameters({ name: f.name, parameters: f.parameters }).equals(selector), + ); + return Promise.resolve(func?.name); + } } diff --git a/yarn-project/circuit-types/src/interfaces/archiver.test.ts b/yarn-project/circuit-types/src/interfaces/archiver.test.ts index 3c5385bb5e57..5eedfe9e5853 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.test.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.test.ts @@ -365,6 +365,15 @@ class MockArchiver implements ArchiverApi { const contractClass = getContractClassFromArtifact(this.artifact); return Promise.resolve(computePublicBytecodeCommitment(contractClass.packedBytecode)); } + getContractFunctionName(address: AztecAddress, selector: FunctionSelector): Promise { + expect(address).toBeInstanceOf(AztecAddress); + expect(selector).toBeInstanceOf(FunctionSelector); + return Promise.resolve( + this.artifact.functions.find(f => + FunctionSelector.fromNameAndParameters({ name: f.name, parameters: f.parameters }).equals(selector), + )?.name, + ); + } getContract(address: AztecAddress): Promise { return Promise.resolve({ address, diff --git a/yarn-project/circuit-types/src/interfaces/archiver.ts b/yarn-project/circuit-types/src/interfaces/archiver.ts index 10b2edd9b6a4..cfdaafdae74b 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.ts @@ -79,4 +79,8 @@ export const ArchiverApiSchema: ApiSchemaFor = { getL1ToL2MessageIndex: z.function().args(schemas.Fr).returns(schemas.BigInt.optional()), // TODO(#10007): Remove this method addContractClass: z.function().args(ContractClassPublicSchema).returns(z.void()), + getContractFunctionName: z + .function() + .args(schemas.AztecAddress, schemas.FunctionSelector) + .returns(optional(z.string())), }; diff --git a/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts b/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts index bb0e92e40fc5..84ed5f4afd96 100644 --- a/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts +++ b/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts @@ -48,6 +48,8 @@ export interface ContractDataSource { /** Returns a contract artifact. */ getContractArtifact(address: AztecAddress): Promise; + /** Returns a function's name */ + getContractFunctionName(address: AztecAddress, selector: FunctionSelector): Promise; /** Registers a a contract artifact. */ addContractArtifact(address: AztecAddress, contract: ContractArtifact): Promise; } diff --git a/yarn-project/simulator/src/public/public_db_sources.ts b/yarn-project/simulator/src/public/public_db_sources.ts index ea02a4d82aaa..eec8831ac26d 100644 --- a/yarn-project/simulator/src/public/public_db_sources.ts +++ b/yarn-project/simulator/src/public/public_db_sources.ts @@ -14,7 +14,7 @@ import { ContractInstanceDeployedEvent, type ContractInstanceWithAddress, Fr, - FunctionSelector, + type FunctionSelector, type L1_TO_L2_MSG_TREE_HEIGHT, type NULLIFIER_TREE_HEIGHT, type NullifierLeafPreimage, @@ -132,19 +132,7 @@ export class ContractsDataSourcePublicDB implements PublicContractsDB { } public async getDebugFunctionName(address: AztecAddress, selector: FunctionSelector): Promise { - const artifact = await this.dataSource.getContractArtifact(address); - if (!artifact) { - return Promise.resolve(undefined); - } - - const f = artifact.functions.find(f => - FunctionSelector.fromNameAndParameters(f.name, f.parameters).equals(selector), - ); - if (!f) { - return Promise.resolve(undefined); - } - - return Promise.resolve(`${artifact.name}:${f.name}`); + return await this.dataSource.getContractFunctionName(address, selector); } } diff --git a/yarn-project/txe/src/util/txe_public_contract_data_source.ts b/yarn-project/txe/src/util/txe_public_contract_data_source.ts index d38bc684f1a9..a56aeb40d8b2 100644 --- a/yarn-project/txe/src/util/txe_public_contract_data_source.ts +++ b/yarn-project/txe/src/util/txe_public_contract_data_source.ts @@ -74,6 +74,17 @@ export class TXEPublicContractDataSource implements ContractDataSource { return this.txeOracle.getContractDataOracle().getContractArtifact(instance.contractClassId); } + async getContractFunctionName(address: AztecAddress, selector: FunctionSelector): Promise { + const artifact = await this.getContractArtifact(address); + if (!artifact) { + return undefined; + } + const func = artifact.functions.find(f => + FunctionSelector.fromNameAndParameters({ name: f.name, parameters: f.parameters }).equals(selector), + ); + return Promise.resolve(func?.name); + } + addContractArtifact(address: AztecAddress, contract: ContractArtifact): Promise { return this.txeOracle.addContractArtifact(contract); } From 2843505cfc9b03ba88ec3130a7f13ffc04fdfa58 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sat, 23 Nov 2024 18:22:14 +0000 Subject: [PATCH 06/21] WIP --- .../simulator/src/avm/avm_simulator.ts | 67 +++---------------- 1 file changed, 11 insertions(+), 56 deletions(-) diff --git a/yarn-project/simulator/src/avm/avm_simulator.ts b/yarn-project/simulator/src/avm/avm_simulator.ts index 442c3c293ef3..c6f39dc2ccf6 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.ts @@ -6,7 +6,6 @@ import { MAX_L2_GAS_PER_ENQUEUED_CALL, } from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { Timer } from '@aztec/foundation/timer'; import { strict as assert } from 'assert'; @@ -30,7 +29,6 @@ import { type InstructionSet, decodeInstructionFromBytecode, } from './serialization/bytecode_serialization.js'; -import { Opcode } from './serialization/instruction_serialization.js'; type OpcodeTally = { count: number; @@ -48,12 +46,19 @@ export class AvmSimulator { private opcodeTallies: Map = new Map(); private pcTallies: Map = new Map(); + private tallyPrintFunction = () => {}; + private tallyInstructionFunction = (_a: number, _b: string, _c: Gas) => {}; + constructor(private context: AvmContext, private instructionSet: InstructionSet = INSTRUCTION_SET()) { assert( context.machineState.gasLeft.l2Gas <= MAX_L2_GAS_PER_ENQUEUED_CALL, `Cannot allocate more than ${MAX_L2_GAS_PER_ENQUEUED_CALL} to the AVM for execution of an enqueued call`, ); this.log = createDebugLogger(`aztec:avm_simulator:core(f:${context.environment.functionSelector.toString()})`); + if (process.env.LOG_LEVEL === 'debug') { + this.tallyPrintFunction = this.printOpcodeTallies; + this.tallyInstructionFunction = this.tallyInstruction; + } } public static create( @@ -88,9 +93,7 @@ export class AvmSimulator { * Fetch the bytecode and execute it in the current context. */ public async execute(): Promise { - const timer = new Timer(); const bytecode = await this.context.persistableState.getBytecode(this.context.environment.address); - this.log.info(`Retrieved bytecode in ${timer.ms()}ms`); // This assumes that we will not be able to send messages to accounts without code // Pending classes and instances impl details if (!bytecode) { @@ -116,12 +119,6 @@ export class AvmSimulator { assert(bytecode.length > 0, "AVM simulator can't execute empty bytecode"); this.bytecode = bytecode; - let totalExecutionTime = 0; - let totalDecodeTime = 0; - - const executionTimes: number[] = []; - const decodeTimes: number[] = []; - const counts: number[] = []; const { machineState } = this.context; try { @@ -129,25 +126,11 @@ export class AvmSimulator { // continuing until the machine state signifies a halt let instrCounter = 0; while (!machineState.getHalted()) { - let timer = new Timer(); const [instruction, bytesRead] = decodeInstructionFromBytecode(bytecode, machineState.pc, this.instructionSet); assert( !!instruction, 'AVM attempted to execute non-existent instruction. This should never happen (invalid bytecode or AVM simulator bug)!', ); - let duration = timer.ms(); - totalDecodeTime += duration; - if (decodeTimes[instruction.opcode] == undefined) { - decodeTimes[instruction.opcode] = duration; - } else { - decodeTimes[instruction.opcode] += duration; - } - - if (counts[instruction.opcode] == undefined) { - counts[instruction.opcode] = 1; - } else { - counts[instruction.opcode]++; - } const instrStartGas = machineState.gasLeft; // Save gas before executing instruction (for profiling) const instrPc = machineState.pc; // Save PC before executing instruction (for profiling) @@ -161,27 +144,19 @@ export class AvmSimulator { // Normal returns and reverts will return normally here. // "Exceptional halts" will throw. machineState.nextPc = machineState.pc + bytesRead; - timer = new Timer(); + await instruction.execute(this.context); if (!instruction.handlesPC()) { // Increment PC if the instruction doesn't handle it itself machineState.pc += bytesRead; } - duration = timer.ms(); - totalExecutionTime += duration; - - if (executionTimes[instruction.opcode] == undefined) { - executionTimes[instruction.opcode] = duration; - } else { - executionTimes[instruction.opcode] += duration; - } // gas used by this instruction - used for profiling/tallying const gasUsed: Gas = { l2Gas: instrStartGas.l2Gas - machineState.l2GasLeft, daGas: instrStartGas.daGas - machineState.daGasLeft, }; - this.tallyInstruction(instrPc, instruction.constructor.name, gasUsed); + this.tallyInstructionFunction(instrPc, instruction.constructor.name, gasUsed); if (machineState.pc >= bytecode.length) { this.log.warn('Passed end of program'); @@ -195,27 +170,7 @@ export class AvmSimulator { const results = new AvmContractCallResult(reverted, output, machineState.gasLeft, revertReason); this.log.debug(`Context execution results: ${results.toString()}`); - this.log.info( - `Total decode time ${totalDecodeTime}ms, total execution time ${totalExecutionTime}ms, num instructions ${instrCounter}`, - ); - - for (let i = 0; i < decodeTimes.length; i++) { - if (decodeTimes[i] == undefined) { - continue; - } - this.log.info(`Decode Opcode ${Opcode[i]}: ${decodeTimes[i]} / ${counts[i]} = ${decodeTimes[i] / counts[i]}`); - } - - for (let i = 0; i < executionTimes.length; i++) { - if (executionTimes[i] == undefined) { - continue; - } - this.log.info( - `Execution Opcode ${Opcode[i]}: ${executionTimes[i]} / ${counts[i]} = ${executionTimes[i] / counts[i]}`, - ); - } - - this.printOpcodeTallies(); + this.tallyPrintFunction(); // Return results for processing by calling context return results; } catch (err: any) { @@ -230,7 +185,7 @@ export class AvmSimulator { const results = new AvmContractCallResult(/*reverted=*/ true, /*output=*/ [], machineState.gasLeft, revertReason); this.log.debug(`Context execution results: ${results.toString()}`); - this.printOpcodeTallies(); + this.tallyPrintFunction(); // Return results for processing by calling context return results; } From 0da28dc7c4bac439872f37d93ae4e501b43354ee Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 24 Nov 2024 23:14:47 +0000 Subject: [PATCH 07/21] WIP --- cspell.json | 1 + .../kv_archiver_store/kv_archiver_store.ts | 3 --- .../simulator/src/avm/journal/journal.ts | 15 ----------- .../simulator/src/public/public_processor.ts | 7 ++++- .../src/public/public_tx_simulator.ts | 12 --------- .../src/native/native_world_state.ts | 26 ++++++++++++++----- .../server_world_state_synchronizer.ts | 19 +++++++++++--- 7 files changed, 42 insertions(+), 41 deletions(-) diff --git a/cspell.json b/cspell.json index 6073069b0288..6a62badf5be6 100644 --- a/cspell.json +++ b/cspell.json @@ -228,6 +228,7 @@ "rollups", "rushstack", "sanitise", + "sanitised", "schnorr", "secp", "SEMRESATTRS", diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 6a03c79869b3..1d009ab18931 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -85,10 +85,7 @@ export class KVArchiverDataStore implements ArchiverDataStore { } getContractInstance(address: AztecAddress): Promise { - const timer = new Timer(); const contract = this.#contractInstanceStore.getContractInstance(address); - this.#log.info(`Retrieved contract in ${timer.ms()}ms`); - return Promise.resolve(contract); } diff --git a/yarn-project/simulator/src/avm/journal/journal.ts b/yarn-project/simulator/src/avm/journal/journal.ts index 446148ad861c..d46acd4ef6af 100644 --- a/yarn-project/simulator/src/avm/journal/journal.ts +++ b/yarn-project/simulator/src/avm/journal/journal.ts @@ -469,21 +469,12 @@ export class AvmPersistableStateManager { * Get a contract's bytecode from the contracts DB, also trace the contract class and instance */ public async getBytecode(contractAddress: AztecAddress): Promise { - this.log.debug(`Getting bytecode for contract address ${contractAddress}`); - let timer = new Timer(); const instanceWithAddress = await this.worldStateDB.getContractInstance(contractAddress); const exists = instanceWithAddress !== undefined; - let duration = timer.ms(); - this.log.info(`Retrieved contract in ${duration}ms`); if (exists) { const instance = new SerializableContractInstance(instanceWithAddress); - - timer = new Timer(); const contractClass = await this.worldStateDB.getContractClass(instance.contractClassId); - duration = timer.ms(); - this.log.info(`Retrieved contract class in ${duration}ms`); - const bytecodeCommitment = await this.worldStateDB.getBytecodeCommitment(instance.contractClassId); assert( @@ -496,16 +487,12 @@ export class AvmPersistableStateManager { `Bytecode commitment was not found in DB for contract class (${instance.contractClassId}). This should not happen!`, ); - timer = new Timer(); const contractClassPreimage = { artifactHash: contractClass.artifactHash, privateFunctionsRoot: contractClass.privateFunctionsRoot, publicBytecodeCommitment: bytecodeCommitment, }; - duration = timer.ms(); - this.log.info(`Computed bytecode commitment in ${duration}ms`); - timer = new Timer(); this.trace.traceGetBytecode( contractAddress, exists, @@ -513,8 +500,6 @@ export class AvmPersistableStateManager { instance, contractClassPreimage, ); - duration = timer.ms(); - this.log.info(`Retrieved bytecode in ${duration}ms`); return contractClass.packedBytecode; } else { diff --git a/yarn-project/simulator/src/public/public_processor.ts b/yarn-project/simulator/src/public/public_processor.ts index 7e1e35b8710b..516f30e956ac 100644 --- a/yarn-project/simulator/src/public/public_processor.ts +++ b/yarn-project/simulator/src/public/public_processor.ts @@ -255,6 +255,8 @@ export class PublicProcessor { const { avmProvingRequest, gasUsed, revertCode, revertReason, processedPhases } = await this.publicTxSimulator.simulate(tx); + this.log.info(`DONE in ${timer.ms()}`); + if (!avmProvingRequest) { this.metrics.recordFailedTx(); throw new Error('Avm proving result was not generated.'); @@ -276,7 +278,10 @@ export class PublicProcessor { ); const phaseCount = processedPhases.length; - this.metrics.recordTx(phaseCount, timer.ms()); + const durationMs = timer.ms(); + this.metrics.recordTx(phaseCount, durationMs); + const durationSeconds = durationMs / 1000; + this.log.info(`SIMULATED TX in ${durationMs}ms, that's ${gasUsed.totalGas.l2Gas / durationSeconds} per second`); const data = avmProvingRequest.inputs.output; const feePaymentPublicDataWrite = await this.getFeePaymentPublicDataWrite( diff --git a/yarn-project/simulator/src/public/public_tx_simulator.ts b/yarn-project/simulator/src/public/public_tx_simulator.ts index 847cf5bb01f5..7a05b4edd69a 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.ts @@ -73,14 +73,8 @@ export class PublicTxSimulator { async simulate(tx: Tx): Promise { this.log.verbose(`Processing tx ${tx.getTxHash()}`); - let timer = new Timer(); - const context = await PublicTxContext.create(this.db, this.worldStateDB, tx, this.globalVariables); - this.log.info(`Created context in ${timer.ms()}ms`); - - timer = new Timer(); - // add new contracts to the contracts db so that their functions may be found and called // TODO(#4073): This is catching only private deployments, when we add public ones, we'll // have to capture contracts emitted in that phase as well. @@ -94,20 +88,14 @@ export class PublicTxSimulator { if (context.hasPhase(TxExecutionPhase.SETUP)) { const setupResult: ProcessedPhase = await this.simulateSetupPhase(context); processedPhases.push(setupResult); - this.log.info(`Simulated setup in ${timer.ms()}ms`); - timer = new Timer(); } if (context.hasPhase(TxExecutionPhase.APP_LOGIC)) { const appLogicResult: ProcessedPhase = await this.simulateAppLogicPhase(context); processedPhases.push(appLogicResult); - this.log.info(`Simulated app logic in ${timer.ms()}ms`); - timer = new Timer(); } if (context.hasPhase(TxExecutionPhase.TEARDOWN)) { const teardownResult: ProcessedPhase = await this.simulateTeardownPhase(context); processedPhases.push(teardownResult); - this.log.info(`Simulated teardown in ${timer.ms()}ms`); - timer = new Timer(); } context.halt(); diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index 8e159ba5606b..b4b43e9041f6 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -32,6 +32,7 @@ import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkl import { WorldStateMessageType, type WorldStateStatusFull, + WorldStateStatusSummary, blockStateReference, sanitiseFullStatus, sanitiseSummary, @@ -52,6 +53,8 @@ export const WORLD_STATE_DB_VERSION = 1; // The initial version export class NativeWorldStateService implements MerkleTreeDatabase { protected initialHeader: Header | undefined; + // This is read heavily and only changes when data is persisted, so we cache it + private cachedStatusSummary: WorldStateStatusSummary | undefined; protected constructor( protected readonly instance: NativeWorldState, @@ -197,7 +200,9 @@ export class NativeWorldStateService implements MerkleTreeDatabase { batchesOfPublicDataWrites: batchesOfPublicDataWrites.map(batch => batch.map(serializeLeaf)), blockStateRef: blockStateReference(l2Block.header.state), }); - return sanitiseFullStatus(response); + const sanitised = sanitiseFullStatus(response); + this.cachedStatusSummary = { ...sanitised.summary }; + return sanitised; } public async close(): Promise { @@ -219,7 +224,8 @@ export class NativeWorldStateService implements MerkleTreeDatabase { const response = await this.instance.call(WorldStateMessageType.FINALISE_BLOCKS, { toBlockNumber, }); - return sanitiseSummary(response); + this.cachedStatusSummary = sanitiseSummary(response); + return this.getStatusSummary(); } /** @@ -231,7 +237,9 @@ export class NativeWorldStateService implements MerkleTreeDatabase { const response = await this.instance.call(WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS, { toBlockNumber, }); - return sanitiseFullStatus(response); + const sanitised = sanitiseFullStatus(response); + this.cachedStatusSummary = { ...sanitised.summary }; + return sanitised; } /** @@ -243,12 +251,18 @@ export class NativeWorldStateService implements MerkleTreeDatabase { const response = await this.instance.call(WorldStateMessageType.UNWIND_BLOCKS, { toBlockNumber, }); - return sanitiseFullStatus(response); + const sanitised = sanitiseFullStatus(response); + this.cachedStatusSummary = { ...sanitised.summary }; + return sanitised; } public async getStatusSummary() { - const response = await this.instance.call(WorldStateMessageType.GET_STATUS, void 0); - return sanitiseSummary(response); + if (this.cachedStatusSummary === undefined) { + const response = await this.instance.call(WorldStateMessageType.GET_STATUS, void 0); + this.cachedStatusSummary = sanitiseSummary(response); + } + + return { ...this.cachedStatusSummary }; } updateLeaf( diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index 1678b22e41a7..35feaa4563e5 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -40,6 +40,7 @@ export class ServerWorldStateSynchronizer private latestBlockNumberAtStart = 0; private currentState: WorldStateRunningState = WorldStateRunningState.IDLE; + private latestBlockHashQuery: { blockNumber: number; hash: string | undefined } | undefined = undefined; private syncPromise = promiseWithResolvers(); protected blockStream: L2BlockStream | undefined; @@ -155,10 +156,19 @@ export class ServerWorldStateSynchronizer } /** Returns the L2 block hash for a given number. Used by the L2BlockStream for detecting reorgs. */ - public getL2BlockHash(number: number): Promise { - return number === 0 - ? Promise.resolve(this.merkleTreeCommitted.getInitialHeader().hash().toString()) - : this.merkleTreeCommitted.getLeafValue(MerkleTreeId.ARCHIVE, BigInt(number)).then(leaf => leaf?.toString()); + public async getL2BlockHash(number: number): Promise { + if (number === 0) { + return Promise.resolve(this.merkleTreeCommitted.getInitialHeader().hash().toString()); + } + if (this.latestBlockHashQuery?.hash === undefined || number !== this.latestBlockHashQuery.blockNumber) { + this.latestBlockHashQuery = { + hash: await this.merkleTreeCommitted + .getLeafValue(MerkleTreeId.ARCHIVE, BigInt(number)) + .then(leaf => leaf?.toString()), + blockNumber: number, + }; + } + return this.latestBlockHashQuery.hash; } /** Returns the latest L2 block number for each tip of the chain (latest, proven, finalized). */ @@ -256,6 +266,7 @@ export class ServerWorldStateSynchronizer private async handleChainPruned(blockNumber: number) { this.log.info(`Chain pruned to block ${blockNumber}`); await this.merkleTreeDb.unwindBlocks(BigInt(blockNumber)); + this.latestBlockHashQuery = undefined; } /** From b9dc6ac2b5a598fdfcf1658cc3f8f0991ec748bf Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 24 Nov 2024 23:30:23 +0000 Subject: [PATCH 08/21] WIP --- yarn-project/archiver/src/archiver/archiver.ts | 1 + .../src/archiver/kv_archiver_store/kv_archiver_store.ts | 2 +- yarn-project/foundation/src/crypto/poseidon/index.ts | 9 +-------- yarn-project/simulator/src/avm/journal/journal.ts | 3 +-- yarn-project/simulator/src/public/public_db_sources.ts | 3 +-- yarn-project/simulator/src/public/public_processor.ts | 4 ---- yarn-project/simulator/src/public/public_tx_simulator.ts | 6 +----- 7 files changed, 6 insertions(+), 22 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 53564a70c1d9..a19990d06d0e 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -834,6 +834,7 @@ class ArchiverStoreHelper if (contractClasses.length > 0) { contractClasses.forEach(c => this.#log.verbose(`Registering contract class ${c.id.toString()}`)); if (operation == Operation.Store) { + // TODO: Will probably want to create some worker threads to compute these bytecode commitments as they are expensive return await this.store.addContractClasses( contractClasses, contractClasses.map(x => computePublicBytecodeCommitment(x.packedBytecode)), diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 1d009ab18931..111a4e986032 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -22,7 +22,6 @@ import { import { type ContractArtifact, FunctionSelector } from '@aztec/foundation/abi'; import { type AztecAddress } from '@aztec/foundation/aztec-address'; import { createDebugLogger } from '@aztec/foundation/log'; -import { Timer } from '@aztec/foundation/timer'; import { type AztecKVStore } from '@aztec/kv-store'; import { type ArchiverDataStore, type ArchiverL1SynchPoint } from '../archiver_store.js'; @@ -71,6 +70,7 @@ export class KVArchiverDataStore implements ArchiverDataStore { async addContractArtifact(address: AztecAddress, contract: ContractArtifact): Promise { await this.#contractArtifactStore.addContractArtifact(address, contract); + // Building tup this map of selectors to function names save expensive re-hydration of contract artifacts later contract.functions.forEach(f => { this.functionNames.set(FunctionSelector.fromNameAndParameters(f.name, f.parameters).toString(), f.name); }); diff --git a/yarn-project/foundation/src/crypto/poseidon/index.ts b/yarn-project/foundation/src/crypto/poseidon/index.ts index 53aee7f64b64..aad83209f2f6 100644 --- a/yarn-project/foundation/src/crypto/poseidon/index.ts +++ b/yarn-project/foundation/src/crypto/poseidon/index.ts @@ -41,19 +41,12 @@ export function poseidon2HashWithSeparator(input: Fieldable[], separator: number ); } -/** - * Create a poseidon hash (field) from an array of input fields and a domain separator. - * @param input - The input fields to hash. - * @returns The poseidon hash. - */ export function poseidon2HashAccumulate(input: Fieldable[]): Fr { const inputFields = serializeToFields(input); return Fr.fromBuffer( Buffer.from( BarretenbergSync.getSingleton() - .poseidon2HashAccumulate( - inputFields.map(i => new FrBarretenberg(i.toBuffer())), // TODO(#4189): remove this stupid conversion - ) + .poseidon2HashAccumulate(inputFields.map(i => new FrBarretenberg(i.toBuffer()))) .toBuffer(), ), ); diff --git a/yarn-project/simulator/src/avm/journal/journal.ts b/yarn-project/simulator/src/avm/journal/journal.ts index d46acd4ef6af..3e66b75a0206 100644 --- a/yarn-project/simulator/src/avm/journal/journal.ts +++ b/yarn-project/simulator/src/avm/journal/journal.ts @@ -7,12 +7,10 @@ import { PublicDataTreeLeaf, PublicDataTreeLeafPreimage, SerializableContractInstance, - computePublicBytecodeCommitment, } from '@aztec/circuits.js'; import { computePublicDataTreeLeafSlot, siloNoteHash, siloNullifier } from '@aztec/circuits.js/hash'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; -import { Timer } from '@aztec/foundation/timer'; import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import assert from 'assert'; @@ -469,6 +467,7 @@ export class AvmPersistableStateManager { * Get a contract's bytecode from the contracts DB, also trace the contract class and instance */ public async getBytecode(contractAddress: AztecAddress): Promise { + this.log.debug(`Getting bytecode for contract address ${contractAddress}`); const instanceWithAddress = await this.worldStateDB.getContractInstance(contractAddress); const exists = instanceWithAddress !== undefined; diff --git a/yarn-project/simulator/src/public/public_db_sources.ts b/yarn-project/simulator/src/public/public_db_sources.ts index eec8831ac26d..2984835828b2 100644 --- a/yarn-project/simulator/src/public/public_db_sources.ts +++ b/yarn-project/simulator/src/public/public_db_sources.ts @@ -53,8 +53,7 @@ export class ContractsDataSourcePublicDB implements PublicContractsDB { const logs = tx.contractClassLogs.unrollLogs(); ContractClassRegisteredEvent.fromLogs(logs, ProtocolContractAddress.ContractClassRegisterer).forEach(e => { this.log.debug(`Adding class ${e.contractClassId.toString()} to public execution contract cache`); - const key = e.contractClassId.toString(); - this.classCache.set(key, e.toContractClassPublic()); + this.classCache.set(e.contractClassId.toString(), e.toContractClassPublic()); }); // We store the contract instance deployed event log in enc logs, contract_instance_deployer_contract/src/main.nr const encLogs = tx.encryptedLogs.unrollLogs(); diff --git a/yarn-project/simulator/src/public/public_processor.ts b/yarn-project/simulator/src/public/public_processor.ts index 516f30e956ac..1ef3c4bb8a8f 100644 --- a/yarn-project/simulator/src/public/public_processor.ts +++ b/yarn-project/simulator/src/public/public_processor.ts @@ -255,8 +255,6 @@ export class PublicProcessor { const { avmProvingRequest, gasUsed, revertCode, revertReason, processedPhases } = await this.publicTxSimulator.simulate(tx); - this.log.info(`DONE in ${timer.ms()}`); - if (!avmProvingRequest) { this.metrics.recordFailedTx(); throw new Error('Avm proving result was not generated.'); @@ -280,8 +278,6 @@ export class PublicProcessor { const phaseCount = processedPhases.length; const durationMs = timer.ms(); this.metrics.recordTx(phaseCount, durationMs); - const durationSeconds = durationMs / 1000; - this.log.info(`SIMULATED TX in ${durationMs}ms, that's ${gasUsed.totalGas.l2Gas / durationSeconds} per second`); const data = avmProvingRequest.inputs.output; const feePaymentPublicDataWrite = await this.getFeePaymentPublicDataWrite( diff --git a/yarn-project/simulator/src/public/public_tx_simulator.ts b/yarn-project/simulator/src/public/public_tx_simulator.ts index 7a05b4edd69a..72b84e908d03 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.ts @@ -258,8 +258,6 @@ export class PublicTxSimulator { /*l2Gas=*/ Math.min(availableGas.l2Gas, MAX_L2_GAS_PER_ENQUEUED_CALL), ); - const timer = new Timer(); - const result = await this.simulateEnqueuedCallInternal( context.state.getActiveStateManager(), executionRequest, @@ -272,9 +270,7 @@ export class PublicTxSimulator { const gasUsed = allocatedGas.sub(result.gasLeft); context.consumeGas(phase, gasUsed); this.log.verbose( - `[AVM] Enqueued public call consumed ${gasUsed.l2Gas} L2 gas ending with ${ - result.gasLeft.l2Gas - } L2 gas left. Timer ${timer.ms()}ms`, + `[AVM] Enqueued public call consumed ${gasUsed.l2Gas} L2 gas ending with ${result.gasLeft.l2Gas} L2 gas left.`, ); // TODO(dbanks12): remove once AVM proves entire public tx From a0c21fc231f539946ffd213306fe8f28a2f70e76 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 25 Nov 2024 09:29:05 +0000 Subject: [PATCH 09/21] WIP --- .../src/native/native_world_state.ts | 86 ++++++++++++------- .../src/native/native_world_state_instance.ts | 7 +- 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index b4b43e9041f6..e9d25ed033cf 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -191,18 +191,19 @@ export class NativeWorldStateService implements MerkleTreeDatabase { ); } - const response = await this.instance.call(WorldStateMessageType.SYNC_BLOCK, { - blockNumber: l2Block.number, - blockHeaderHash: l2Block.header.hash(), - paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf), - paddedNoteHashes: paddedNoteHashes.map(serializeLeaf), - paddedNullifiers: paddedNullifiers.map(serializeLeaf), - batchesOfPublicDataWrites: batchesOfPublicDataWrites.map(batch => batch.map(serializeLeaf)), - blockStateRef: blockStateReference(l2Block.header.state), - }); - const sanitised = sanitiseFullStatus(response); - this.cachedStatusSummary = { ...sanitised.summary }; - return sanitised; + return await this.instance.call( + WorldStateMessageType.SYNC_BLOCK, + { + blockNumber: l2Block.number, + blockHeaderHash: l2Block.header.hash(), + paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf), + paddedNoteHashes: paddedNoteHashes.map(serializeLeaf), + paddedNullifiers: paddedNullifiers.map(serializeLeaf), + batchesOfPublicDataWrites: batchesOfPublicDataWrites.map(batch => batch.map(serializeLeaf)), + blockStateRef: blockStateReference(l2Block.header.state), + }, + this.sanitiseAndCacheSummary, + ); } public async close(): Promise { @@ -215,16 +216,30 @@ export class NativeWorldStateService implements MerkleTreeDatabase { return Header.empty({ state }); } + private sanitiseAndCacheSummary(response: WorldStateStatusFull) { + const sanitised = sanitiseFullStatus(response); + this.cachedStatusSummary = { ...sanitised.summary }; + return sanitised; + } + /** * Advances the finalised block number to be the number provided * @param toBlockNumber The block number that is now the tip of the finalised chain * @returns The new WorldStateStatus */ public async setFinalised(toBlockNumber: bigint) { - const response = await this.instance.call(WorldStateMessageType.FINALISE_BLOCKS, { - toBlockNumber, - }); - this.cachedStatusSummary = sanitiseSummary(response); + const cacheStatusSummary = (response: WorldStateStatusSummary) => { + const sanitised = sanitiseSummary(response); + this.cachedStatusSummary = { ...sanitised }; + return sanitised; + }; + await this.instance.call( + WorldStateMessageType.FINALISE_BLOCKS, + { + toBlockNumber, + }, + cacheStatusSummary, + ); return this.getStatusSummary(); } @@ -234,12 +249,13 @@ export class NativeWorldStateService implements MerkleTreeDatabase { * @returns The new WorldStateStatus */ public async removeHistoricalBlocks(toBlockNumber: bigint) { - const response = await this.instance.call(WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS, { - toBlockNumber, - }); - const sanitised = sanitiseFullStatus(response); - this.cachedStatusSummary = { ...sanitised.summary }; - return sanitised; + return await this.instance.call( + WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS, + { + toBlockNumber, + }, + this.sanitiseAndCacheSummary, + ); } /** @@ -248,21 +264,25 @@ export class NativeWorldStateService implements MerkleTreeDatabase { * @returns The new WorldStateStatus */ public async unwindBlocks(toBlockNumber: bigint) { - const response = await this.instance.call(WorldStateMessageType.UNWIND_BLOCKS, { - toBlockNumber, - }); - const sanitised = sanitiseFullStatus(response); - this.cachedStatusSummary = { ...sanitised.summary }; - return sanitised; + return await this.instance.call( + WorldStateMessageType.UNWIND_BLOCKS, + { + toBlockNumber, + }, + this.sanitiseAndCacheSummary, + ); } public async getStatusSummary() { - if (this.cachedStatusSummary === undefined) { - const response = await this.instance.call(WorldStateMessageType.GET_STATUS, void 0); - this.cachedStatusSummary = sanitiseSummary(response); + if (this.cachedStatusSummary !== undefined) { + return { ...this.cachedStatusSummary }; } - - return { ...this.cachedStatusSummary }; + const cacheStatusSummary = (response: WorldStateStatusSummary) => { + const sanitised = sanitiseSummary(response); + this.cachedStatusSummary = { ...sanitised }; + return sanitised; + }; + return await this.instance.call(WorldStateMessageType.GET_STATUS, void 0, cacheStatusSummary); } updateLeaf( diff --git a/yarn-project/world-state/src/native/native_world_state_instance.ts b/yarn-project/world-state/src/native/native_world_state_instance.ts index ab63e2d6e5ab..b82de06da314 100644 --- a/yarn-project/world-state/src/native/native_world_state_instance.ts +++ b/yarn-project/world-state/src/native/native_world_state_instance.ts @@ -113,11 +113,14 @@ export class NativeWorldState implements NativeWorldStateInstance { public call( messageType: T, body: WorldStateRequest[T], + // allows for the pre-processing of responses on the job queue before being passed back + responseHandler = (response: WorldStateResponse[T]): WorldStateResponse[T] => response, ): Promise { - return this.queue.put(() => { + return this.queue.put(async () => { assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the native instance'); assert.equal(this.open, true, 'Native instance is closed'); - return this._sendMessage(messageType, body); + const response = await this._sendMessage(messageType, body); + return responseHandler ? responseHandler(response) : response; }); } From f932cf39b07834fa79db1676893e9de18605c55a Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 25 Nov 2024 09:32:24 +0000 Subject: [PATCH 10/21] Formatting --- yarn-project/circuits.js/src/contract/contract_class_id.ts | 2 +- yarn-project/simulator/src/public/public_tx_simulator.test.ts | 2 +- yarn-project/world-state/src/native/native_world_state.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn-project/circuits.js/src/contract/contract_class_id.ts b/yarn-project/circuits.js/src/contract/contract_class_id.ts index af72181b2266..ef401fe5691c 100644 --- a/yarn-project/circuits.js/src/contract/contract_class_id.ts +++ b/yarn-project/circuits.js/src/contract/contract_class_id.ts @@ -1,5 +1,5 @@ import { bufferAsFields } from '@aztec/foundation/abi'; -import { poseidon2Hash, poseidon2HashAccumulate, poseidon2HashWithSeparator } from '@aztec/foundation/crypto'; +import { poseidon2HashAccumulate, poseidon2HashWithSeparator } from '@aztec/foundation/crypto'; import { Fr } from '@aztec/foundation/fields'; import { strict as assert } from 'assert'; diff --git a/yarn-project/simulator/src/public/public_tx_simulator.test.ts b/yarn-project/simulator/src/public/public_tx_simulator.test.ts index 47b2dbc0d219..9e22910cca8f 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.test.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.test.ts @@ -28,7 +28,7 @@ import { type MockProxy, mock } from 'jest-mock-extended'; import { AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmPersistableStateManager } from '../avm/journal/journal.js'; -import { INSTRUCTION_SET, type InstructionSet } from '../avm/serialization/bytecode_serialization.js'; +import { type InstructionSet } from '../avm/serialization/bytecode_serialization.js'; import { type WorldStateDB } from './public_db_sources.js'; import { PublicTxSimulator } from './public_tx_simulator.js'; diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index e9d25ed033cf..65a1c2c2e5a9 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -32,7 +32,7 @@ import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkl import { WorldStateMessageType, type WorldStateStatusFull, - WorldStateStatusSummary, + type WorldStateStatusSummary, blockStateReference, sanitiseFullStatus, sanitiseSummary, From 05159b85b4bb0d2b24726d22f97d9348a68add87 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 25 Nov 2024 13:36:33 +0000 Subject: [PATCH 11/21] Keep instruction set iniitialisation within AVM simulator --- yarn-project/simulator/src/avm/avm_simulator.ts | 2 +- yarn-project/simulator/src/public/public_tx_simulator.ts | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/yarn-project/simulator/src/avm/avm_simulator.ts b/yarn-project/simulator/src/avm/avm_simulator.ts index 831c711a45f0..d6def2e96fcb 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.ts @@ -71,7 +71,6 @@ export class AvmSimulator { isStaticCall: boolean, calldata: Fr[], allocatedGas: Gas, - instructionSet: InstructionSet, ) { const avmExecutionEnv = new AvmExecutionEnvironment( address, @@ -86,6 +85,7 @@ export class AvmSimulator { const avmMachineState = new AvmMachineState(allocatedGas); const avmContext = new AvmContext(stateManager, avmExecutionEnv, avmMachineState); + const instructionSet = INSTRUCTION_SET(); return new AvmSimulator(avmContext, instructionSet); } diff --git a/yarn-project/simulator/src/public/public_tx_simulator.ts b/yarn-project/simulator/src/public/public_tx_simulator.ts index 1b2c43edf28b..44801eff13f8 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.ts @@ -24,7 +24,6 @@ import { type TelemetryClient } from '@aztec/telemetry-client'; import { type AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmPersistableStateManager, AvmSimulator } from '../avm/index.js'; -import { INSTRUCTION_SET, type InstructionSet } from '../avm/serialization/bytecode_serialization.js'; import { getPublicFunctionDebugName } from '../common/debug_fn_name.js'; import { ExecutorMetrics } from './executor_metrics.js'; import { type WorldStateDB } from './public_db_sources.js'; @@ -52,7 +51,6 @@ export class PublicTxSimulator { metrics: ExecutorMetrics; private log: DebugLogger; - private instructionSet: InstructionSet = INSTRUCTION_SET(); constructor( private db: MerkleTreeReadOperations, @@ -271,7 +269,6 @@ export class PublicTxSimulator { allocatedGas, context.getTransactionFee(phase), fnName, - this.instructionSet, ); const gasUsed = allocatedGas.sub(result.gasLeft); @@ -321,7 +318,6 @@ export class PublicTxSimulator { allocatedGas: Gas, transactionFee: Fr, fnName: string, - instructionSet: InstructionSet, ): Promise { const address = executionRequest.callContext.contractAddress; const sender = executionRequest.callContext.msgSender; @@ -342,7 +338,6 @@ export class PublicTxSimulator { executionRequest.callContext.isStaticCall, executionRequest.args, allocatedGas, - instructionSet, ); const avmCallResult = await simulator.execute(); const result = avmCallResult.finalize(); From d01a9e3b78e6821b3b513bbb5810c6075b4dd452 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 25 Nov 2024 13:44:50 +0000 Subject: [PATCH 12/21] Function bind fixes --- yarn-project/world-state/src/native/native_world_state.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index 65a1c2c2e5a9..2acfb1b40970 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -202,7 +202,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { batchesOfPublicDataWrites: batchesOfPublicDataWrites.map(batch => batch.map(serializeLeaf)), blockStateRef: blockStateReference(l2Block.header.state), }, - this.sanitiseAndCacheSummary, + this.sanitiseAndCacheSummary.bind(this), ); } @@ -254,7 +254,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { { toBlockNumber, }, - this.sanitiseAndCacheSummary, + this.sanitiseAndCacheSummary.bind(this), ); } @@ -269,7 +269,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { { toBlockNumber, }, - this.sanitiseAndCacheSummary, + this.sanitiseAndCacheSummary.bind(this), ); } From 8455cbfa926e4ff3b5bc4d7a0d9e1af0f72db84a Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 25 Nov 2024 16:57:03 +0000 Subject: [PATCH 13/21] New test --- .../end-to-end/scripts/e2e_test_config.yml | 2 + .../src/e2e_fees/public_payments.test.ts | 90 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 yarn-project/end-to-end/src/e2e_fees/public_payments.test.ts diff --git a/yarn-project/end-to-end/scripts/e2e_test_config.yml b/yarn-project/end-to-end/scripts/e2e_test_config.yml index b70bd9024d23..1f1b0d58ca8b 100644 --- a/yarn-project/end-to-end/scripts/e2e_test_config.yml +++ b/yarn-project/end-to-end/scripts/e2e_test_config.yml @@ -50,6 +50,8 @@ tests: test_path: 'e2e_fees/gas_estimation.test.ts' e2e_fees_private_payments: test_path: 'e2e_fees/private_payments.test.ts' + e2e_fees_public_payments: + test_path: 'e2e_fees/public_payments.test.ts' e2e_keys: {} e2e_l1_with_wall_time: {} e2e_lending_contract: {} diff --git a/yarn-project/end-to-end/src/e2e_fees/public_payments.test.ts b/yarn-project/end-to-end/src/e2e_fees/public_payments.test.ts new file mode 100644 index 000000000000..1440629072ff --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_fees/public_payments.test.ts @@ -0,0 +1,90 @@ +import { type AccountWallet, type AztecAddress, PublicFeePaymentMethod } from '@aztec/aztec.js'; +import { type GasSettings } from '@aztec/circuits.js'; +import { type TokenContract as BananaCoin, type FPCContract } from '@aztec/noir-contracts.js'; + +import { expectMapping } from '../fixtures/utils.js'; +import { FeesTest } from './fees_test.js'; + +describe('e2e_fees public_payment', () => { + let aliceWallet: AccountWallet; + let aliceAddress: AztecAddress; + let bobAddress: AztecAddress; + let sequencerAddress: AztecAddress; + let bananaCoin: BananaCoin; + let bananaFPC: FPCContract; + let gasSettings: GasSettings; + + const t = new FeesTest('private_payment'); + + beforeAll(async () => { + await t.applyBaseSnapshots(); + await t.applyFPCSetupSnapshot(); + await t.applyFundAliceWithBananas(); + ({ aliceWallet, aliceAddress, bobAddress, sequencerAddress, bananaCoin, bananaFPC, gasSettings } = await t.setup()); + }); + + afterAll(async () => { + await t.teardown(); + }); + + let initialAlicePublicBananas: bigint; + let initialAliceGas: bigint; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + let initialBobPublicBananas: bigint; + + let initialFPCPublicBananas: bigint; + let initialFPCGas: bigint; + + let initialSequencerGas: bigint; + + let maxFee: bigint; + + beforeEach(async () => { + maxFee = BigInt(20e9); + + expect(gasSettings.getFeeLimit().toBigInt()).toEqual(maxFee); + + [ + [initialAlicePublicBananas, initialBobPublicBananas, initialFPCPublicBananas], + [initialAliceGas, initialFPCGas, initialSequencerGas], + ] = await Promise.all([ + t.getBananaPublicBalanceFn(aliceAddress, bobAddress, bananaFPC.address), + t.getGasBalanceFn(aliceAddress, bananaFPC.address, sequencerAddress), + ]); + + // We let Alice see Bob's notes because the expect uses Alice's wallet to interact with the contracts to "get" state. + aliceWallet.setScopes([aliceAddress, bobAddress]); + }); + + it('pays fees for tx that make public transfer', async () => { + const bananasToSendToBob = 10n; + const tx = await bananaCoin.methods + .transfer_in_public(aliceAddress, bobAddress, bananasToSendToBob, 0) + .send({ + fee: { + gasSettings, + paymentMethod: new PublicFeePaymentMethod(bananaCoin.address, bananaFPC.address, aliceWallet), + }, + }) + .wait(); + + const feeAmount = tx.transactionFee!; + + await expectMapping( + t.getBananaPublicBalanceFn, + [aliceAddress, bananaFPC.address, bobAddress], + [ + initialAlicePublicBananas - (feeAmount + bananasToSendToBob), + initialFPCPublicBananas + feeAmount, + initialBobPublicBananas + bananasToSendToBob, + ], + ); + + await expectMapping( + t.getGasBalanceFn, + [aliceAddress, bananaFPC.address, sequencerAddress], + [initialAliceGas, initialFPCGas - feeAmount, initialSequencerGas], + ); + }); +}); From 57f5749a3761b3aedfc1176fbd1e07d40d0567dc Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 25 Nov 2024 22:28:50 +0000 Subject: [PATCH 14/21] Test fixes --- yarn-project/simulator/src/avm/avm_simulator.test.ts | 1 + yarn-project/simulator/src/avm/test_utils.ts | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/yarn-project/simulator/src/avm/avm_simulator.test.ts b/yarn-project/simulator/src/avm/avm_simulator.test.ts index 1c170d7f1c2f..736b0c2b7091 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.test.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.test.ts @@ -6,6 +6,7 @@ import { type PublicFunction, PublicKeys, SerializableContractInstance, + computePublicBytecodeCommitment, } from '@aztec/circuits.js'; import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { computePublicDataTreeLeafSlot, computeVarArgsHash } from '@aztec/circuits.js/hash'; diff --git a/yarn-project/simulator/src/avm/test_utils.ts b/yarn-project/simulator/src/avm/test_utils.ts index 3fa62cddafdf..48afe3c9e324 100644 --- a/yarn-project/simulator/src/avm/test_utils.ts +++ b/yarn-project/simulator/src/avm/test_utils.ts @@ -1,4 +1,9 @@ -import { type ContractClassPublic, type ContractInstanceWithAddress, Fr } from '@aztec/circuits.js'; +import { + type ContractClassPublic, + type ContractInstanceWithAddress, + Fr, + computePublicBytecodeCommitment, +} from '@aztec/circuits.js'; import { type jest } from '@jest/globals'; import { mock } from 'jest-mock-extended'; @@ -8,6 +13,9 @@ import { type PublicSideEffectTraceInterface } from '../public/side_effect_trace export function mockGetBytecode(worldStateDB: WorldStateDB, bytecode: Buffer) { (worldStateDB as jest.Mocked).getBytecode.mockResolvedValue(bytecode); + (worldStateDB as jest.Mocked).getBytecodeCommitment.mockResolvedValue( + computePublicBytecodeCommitment(bytecode), + ); } export function mockTraceFork(trace: PublicSideEffectTraceInterface, nestedTrace?: PublicSideEffectTraceInterface) { From 0cc95a4be49359a606f7bbd90b6c5975fe3df1dd Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 26 Nov 2024 10:54:35 +0000 Subject: [PATCH 15/21] Test fixes --- yarn-project/simulator/src/avm/avm_simulator.test.ts | 1 - yarn-project/simulator/src/public/fixtures/index.ts | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/yarn-project/simulator/src/avm/avm_simulator.test.ts b/yarn-project/simulator/src/avm/avm_simulator.test.ts index 736b0c2b7091..1c170d7f1c2f 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.test.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.test.ts @@ -6,7 +6,6 @@ import { type PublicFunction, PublicKeys, SerializableContractInstance, - computePublicBytecodeCommitment, } from '@aztec/circuits.js'; import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { computePublicDataTreeLeafSlot, computeVarArgsHash } from '@aztec/circuits.js/hash'; diff --git a/yarn-project/simulator/src/public/fixtures/index.ts b/yarn-project/simulator/src/public/fixtures/index.ts index 8c0f53fab32d..40edea8f957c 100644 --- a/yarn-project/simulator/src/public/fixtures/index.ts +++ b/yarn-project/simulator/src/public/fixtures/index.ts @@ -17,6 +17,7 @@ import { SerializableContractInstance, TxConstantData, TxContext, + computePublicBytecodeCommitment, } from '@aztec/circuits.js'; import { makeContractClassPublic, makeContractInstanceFromClassId } from '@aztec/circuits.js/testing'; import { AztecAddress } from '@aztec/foundation/aztec-address'; @@ -84,6 +85,7 @@ export async function simulateAvmTestContractGenerateCircuitInputs( .mockResolvedValue(contractInstance); worldStateDB.getContractClass.mockResolvedValue(contractClass); worldStateDB.getBytecode.mockResolvedValue(bytecode); + worldStateDB.getBytecodeCommitment.mockResolvedValue(computePublicBytecodeCommitment(bytecode)); const storageValue = new Fr(5); worldStateDB.storageRead.mockResolvedValue(Promise.resolve(storageValue)); From 3e5c8c5d231fb9bf8d8faf109734b5dbefd2fad8 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 26 Nov 2024 13:46:22 +0000 Subject: [PATCH 16/21] Review fixes --- yarn-project/simulator/src/avm/serialization/buffer_cursor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/simulator/src/avm/serialization/buffer_cursor.ts b/yarn-project/simulator/src/avm/serialization/buffer_cursor.ts index b4d7ccd76e6f..005a3d6599fd 100644 --- a/yarn-project/simulator/src/avm/serialization/buffer_cursor.ts +++ b/yarn-project/simulator/src/avm/serialization/buffer_cursor.ts @@ -27,7 +27,7 @@ export class BufferCursor { assert(n < this._buffer.length); } - public peakUint8(): number { + public peekUint8(): number { const ret = this._buffer.readUint8(this._position); return ret; } From 332894f3b819da4f04ab5100165e36f28b57aabf Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 26 Nov 2024 13:51:31 +0000 Subject: [PATCH 17/21] Review fixes --- .../src/archiver/kv_archiver_store/kv_archiver_store.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 2bc54eb1f7d6..21567fd329a9 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -64,6 +64,9 @@ export class KVArchiverDataStore implements ArchiverDataStore { return Promise.resolve(this.#contractArtifactStore.getContractArtifact(address)); } + // TODO: These function names are in memory only as they are for development/debugging. They require the full contract + // artifact supplied to the node out of band. This should be reviewed and potentially removed as part of + // the node api cleanup process. getContractFunctionName(address: AztecAddress, selector: FunctionSelector): Promise { return Promise.resolve(this.functionNames.get(selector.toString())); } From 1634e92031e6319c272af8109c00a89f98a0846d Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 26 Nov 2024 17:39:09 +0000 Subject: [PATCH 18/21] Remove historical blocks based on configuration value --- .../src/barretenberg/world_state/world_state.cpp | 7 +++++-- .../src/composed/integration_l1_publisher.test.ts | 1 + yarn-project/foundation/src/config/env_var.ts | 1 + .../src/native/native_world_state.test.ts | 4 ++-- .../world-state/src/synchronizer/config.ts | 10 +++++++++- .../server_world_state_synchronizer.test.ts | 1 + .../server_world_state_synchronizer.ts | 14 +++++++++++++- .../world-state/src/test/integration.test.ts | 1 + 8 files changed, 33 insertions(+), 6 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index a61b0fe19141..eb3f047b2f32 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -639,12 +639,15 @@ WorldStateStatusFull WorldState::remove_historical_blocks(const index_t& toBlock WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; TreeMetaResponse archive_state = get_tree_info(revision, MerkleTreeId::ARCHIVE); if (toBlockNumber <= archive_state.meta.oldestHistoricBlock) { - throw std::runtime_error("Unable to remove historical block, block not found"); + throw std::runtime_error(format("Unable to remove historical blocks to block number ", + toBlockNumber, + ", blocks not found. Current oldest block: ", + archive_state.meta.oldestHistoricBlock)); } WorldStateStatusFull status; for (index_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; blockNumber++) { if (!remove_historical_block(blockNumber, status)) { - throw std::runtime_error("Failed to remove historical block"); + throw std::runtime_error(format("Failed to remove historical block", toBlockNumber)); } } populate_status_summary(status); diff --git a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts index d2775d2eda40..f87146bfe297 100644 --- a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts @@ -145,6 +145,7 @@ describe('L1Publisher integration', () => { worldStateBlockCheckIntervalMS: 10000, worldStateProvenBlocksOnly: false, worldStateDbMapSizeKb: 10 * 1024 * 1024, + worldStateBlockHistory: 0, }; worldStateSynchronizer = new ServerWorldStateSynchronizer( builderDb, diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 4a2015ddd0f8..165d98d7b6ce 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -155,6 +155,7 @@ export type EnvVar = | 'SEQ_VIEM_POLLING_INTERVAL_MS' | 'WS_DB_MAP_SIZE_KB' | 'WS_DATA_DIRECTORY' + | 'WS_NUM_HISTORIC_BLOCKS' | 'ETHEREUM_SLOT_DURATION' | 'AZTEC_SLOT_DURATION' | 'AZTEC_EPOCH_DURATION' diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index dbfd92a0b249..2bf1903e3601 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -343,9 +343,9 @@ describe('NativeWorldState', () => { } //can't prune what has already been pruned - for (let i = 0; i < highestPrunedBlockNumber; i++) { + for (let i = 0; i <= highestPrunedBlockNumber; i++) { await expect(ws.removeHistoricalBlocks(BigInt(i + 1))).rejects.toThrow( - 'Unable to remove historical block, block not found', + `Unable to remove historical blocks to block number ${BigInt(i + 1)}, blocks not found. Current oldest block: ${highestPrunedBlockNumber + 1}` ); } }, 30_000); diff --git a/yarn-project/world-state/src/synchronizer/config.ts b/yarn-project/world-state/src/synchronizer/config.ts index bdd96365da96..465afc6baf73 100644 --- a/yarn-project/world-state/src/synchronizer/config.ts +++ b/yarn-project/world-state/src/synchronizer/config.ts @@ -1,4 +1,4 @@ -import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings } from '@aztec/foundation/config'; +import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config'; /** World State synchronizer configuration values. */ export interface WorldStateConfig { @@ -16,6 +16,9 @@ export interface WorldStateConfig { /** Optional directory for the world state DB, if unspecified will default to the general data directory */ worldStateDataDirectory?: string; + + /** The number of historic blocks to maintain */ + worldStateBlockHistory: number; } export const worldStateConfigMappings: ConfigMappingsType = { @@ -44,6 +47,11 @@ export const worldStateConfigMappings: ConfigMappingsType = { env: 'WS_DATA_DIRECTORY', description: 'Optional directory for the world state database', }, + worldStateBlockHistory: { + env: 'WS_NUM_HISTORIC_BLOCKS', + description: 'The number of historic blocks to maintain. Values less than 1 mean all history is maintained', + ...numberConfigHelper(64), + } }; /** diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts index 2cdcd42e1e10..08023782278a 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts @@ -76,6 +76,7 @@ describe('ServerWorldStateSynchronizer', () => { worldStateBlockCheckIntervalMS: 100, worldStateProvenBlocksOnly: false, worldStateDbMapSizeKb: 1024 * 1024, + worldStateBlockHistory: 0, }; server = new TestWorldStateSynchronizer(merkleTreeDb, blockAndMessagesSource, config, l2BlockStream); diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index 086c7cf534cd..b708adaa4446 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -41,6 +41,7 @@ export class ServerWorldStateSynchronizer private readonly merkleTreeCommitted: MerkleTreeReadOperations; private latestBlockNumberAtStart = 0; + private historyToKeep: number | undefined; private currentState: WorldStateRunningState = WorldStateRunningState.IDLE; private latestBlockHashQuery: { blockNumber: number; hash: string | undefined } | undefined = undefined; @@ -57,6 +58,8 @@ export class ServerWorldStateSynchronizer ) { this.instrumentation = new WorldStateInstrumentation(telemetry); this.merkleTreeCommitted = this.merkleTreeDb.getCommitted(); + this.historyToKeep = config.worldStateBlockHistory < 1 ? undefined : config.worldStateBlockHistory; + this.log.info(`Created world state synchroniser with block history of ${this.historyToKeep === undefined ? 'infinity' : this.historyToKeep}`); } public getCommitted(): MerkleTreeReadOperations { @@ -266,7 +269,16 @@ export class ServerWorldStateSynchronizer private async handleChainFinalized(blockNumber: number) { this.log.verbose(`Chain finalized at block ${blockNumber}`); - await this.merkleTreeDb.setFinalised(BigInt(blockNumber)); + const summary = await this.merkleTreeDb.setFinalised(BigInt(blockNumber)); + if (this.historyToKeep === undefined) { + return; + } + const newHistoricBlock = (summary.finalisedBlockNumber - BigInt(this.historyToKeep)) + 1n; + if (newHistoricBlock <= 1) { + return; + } + this.log.verbose(`Pruning historic blocks to ${newHistoricBlock}`); + await this.merkleTreeDb.removeHistoricalBlocks(newHistoricBlock); } private handleChainProven(blockNumber: number) { diff --git a/yarn-project/world-state/src/test/integration.test.ts b/yarn-project/world-state/src/test/integration.test.ts index 20093e48e187..415f967047ac 100644 --- a/yarn-project/world-state/src/test/integration.test.ts +++ b/yarn-project/world-state/src/test/integration.test.ts @@ -45,6 +45,7 @@ describe('world-state integration', () => { worldStateProvenBlocksOnly: false, worldStateBlockRequestBatchSize: 5, worldStateDbMapSizeKb: 1024 * 1024, + worldStateBlockHistory: 0, }; archiver = new MockPrefilledArchiver(blocks, messages); From b8548ec336d8bd2a78235d08b12890d07aa2a15e Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 26 Nov 2024 17:51:56 +0000 Subject: [PATCH 19/21] More logging --- barretenberg/cpp/src/barretenberg/world_state/world_state.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index eb3f047b2f32..6b688aab92c0 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -647,7 +647,8 @@ WorldStateStatusFull WorldState::remove_historical_blocks(const index_t& toBlock WorldStateStatusFull status; for (index_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; blockNumber++) { if (!remove_historical_block(blockNumber, status)) { - throw std::runtime_error(format("Failed to remove historical block", toBlockNumber)); + throw std::runtime_error(format( + "Failed to remove historical block ", blockNumber, " when removing blocks up to ", toBlockNumber)); } } populate_status_summary(status); From 2c41dcb4f7e17e479a4a2c1c7e7cddd8179d47a1 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 27 Nov 2024 10:32:23 +0000 Subject: [PATCH 20/21] Formatting --- .../world-state/src/native/native_world_state.test.ts | 4 +++- yarn-project/world-state/src/synchronizer/config.ts | 9 +++++++-- .../src/synchronizer/server_world_state_synchronizer.ts | 8 ++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 7cdb00cb3e34..4aaa33b4f914 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -348,7 +348,9 @@ describe('NativeWorldState', () => { //can't prune what has already been pruned for (let i = 0; i <= highestPrunedBlockNumber; i++) { await expect(ws.removeHistoricalBlocks(BigInt(i + 1))).rejects.toThrow( - `Unable to remove historical blocks to block number ${BigInt(i + 1)}, blocks not found. Current oldest block: ${highestPrunedBlockNumber + 1}` + `Unable to remove historical blocks to block number ${BigInt( + i + 1, + )}, blocks not found. Current oldest block: ${highestPrunedBlockNumber + 1}`, ); } }); diff --git a/yarn-project/world-state/src/synchronizer/config.ts b/yarn-project/world-state/src/synchronizer/config.ts index 465afc6baf73..3714126cb5c8 100644 --- a/yarn-project/world-state/src/synchronizer/config.ts +++ b/yarn-project/world-state/src/synchronizer/config.ts @@ -1,4 +1,9 @@ -import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config'; +import { + type ConfigMappingsType, + booleanConfigHelper, + getConfigFromMappings, + numberConfigHelper, +} from '@aztec/foundation/config'; /** World State synchronizer configuration values. */ export interface WorldStateConfig { @@ -51,7 +56,7 @@ export const worldStateConfigMappings: ConfigMappingsType = { env: 'WS_NUM_HISTORIC_BLOCKS', description: 'The number of historic blocks to maintain. Values less than 1 mean all history is maintained', ...numberConfigHelper(64), - } + }, }; /** diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index b708adaa4446..ae344f4144a5 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -59,7 +59,11 @@ export class ServerWorldStateSynchronizer this.instrumentation = new WorldStateInstrumentation(telemetry); this.merkleTreeCommitted = this.merkleTreeDb.getCommitted(); this.historyToKeep = config.worldStateBlockHistory < 1 ? undefined : config.worldStateBlockHistory; - this.log.info(`Created world state synchroniser with block history of ${this.historyToKeep === undefined ? 'infinity' : this.historyToKeep}`); + this.log.info( + `Created world state synchroniser with block history of ${ + this.historyToKeep === undefined ? 'infinity' : this.historyToKeep + }`, + ); } public getCommitted(): MerkleTreeReadOperations { @@ -273,7 +277,7 @@ export class ServerWorldStateSynchronizer if (this.historyToKeep === undefined) { return; } - const newHistoricBlock = (summary.finalisedBlockNumber - BigInt(this.historyToKeep)) + 1n; + const newHistoricBlock = summary.finalisedBlockNumber - BigInt(this.historyToKeep) + 1n; if (newHistoricBlock <= 1) { return; } From 409d59986f583bb3c5fd245a16e641a15b9a2989 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 27 Nov 2024 12:27:48 +0000 Subject: [PATCH 21/21] Fixed merge error --- yarn-project/archiver/src/archiver/archiver_store.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 0352e662183c..27958e22d80e 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -270,7 +270,6 @@ export interface ArchiverDataStore { addContractArtifact(address: AztecAddress, contract: ContractArtifact): Promise; getContractArtifact(address: AztecAddress): Promise; - getContractFunctionName(address: AztecAddress, selector: FunctionSelector): Promise; // TODO: These function names are in memory only as they are for development/debugging. They require the full contract // artifact supplied to the node out of band. This should be reviewed and potentially removed as part of