From 0ea6c896ee6600d62b20af73aae805b6c44dc118 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 1 Apr 2024 11:52:25 +0100 Subject: [PATCH 01/24] WIP --- .../src/orchestrator/orchestrator.test.ts | 213 +++++++++++++++--- .../src/orchestrator/orchestrator.ts | 213 +++++++++++------- .../src/orchestrator/proving-state.ts | 92 +++----- 3 files changed, 343 insertions(+), 175 deletions(-) diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts index c4af9a479dc6..409b572bd474 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts @@ -2,6 +2,7 @@ import { MerkleTreeId, PROVING_STATUS, ProcessedTx, + ProvingFailure, ProvingSuccess, makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, makeProcessedTx, @@ -57,6 +58,9 @@ import { getVerificationKeys } from '../mocks/verification_keys.js'; import { RollupProver } from '../prover/index.js'; import { RollupSimulator } from '../simulator/rollup.js'; import { ProvingOrchestrator } from './orchestrator.js'; +import { createDebugLogger } from '@aztec/foundation/log'; + +const logger = createDebugLogger('aztec:orchestrator-test'); export const createMemDown = () => (memdown as any)() as MemDown; @@ -83,9 +87,13 @@ describe('prover/tx-prover', () => { const coinbase = EthAddress.ZERO; const feeRecipient = AztecAddress.ZERO; + const makeGlobals = (blockNumber: number) => { + return new GlobalVariables(chainId, version, new Fr(blockNumber), Fr.ZERO, coinbase, feeRecipient); + } + beforeEach(async () => { blockNumber = 3; - globalVariables = new GlobalVariables(chainId, version, new Fr(blockNumber), Fr.ZERO, coinbase, feeRecipient); + globalVariables = makeGlobals(blockNumber); builderDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); expectsDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); @@ -436,17 +444,53 @@ describe('prover/tx-prover', () => { it('builds a block with 1 transaction', async () => { const txs = await Promise.all([makeEmptyProcessedTx()]); - const blockTicket = await builder.startNewBlock(txs.length, globalVariables, [], await makeEmptyProcessedTx()); + // This will need to be a 2 tx block + const blockTicket = await builder.startNewBlock(2, globalVariables, [], await makeEmptyProcessedTx()); for (const tx of txs) { await builder.addNewTx(tx); } + // we need to complete the block as we have not added a full set of txs + await builder.setBlockCompleted(); + const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); expect((result as ProvingSuccess).block.number).toEqual(blockNumber); }, 30_000); + it('builds multiple blocks in sequence', async () => { + const numBlocks = 5; + let header = await builderDb.buildInitialHeader(); + + for (let i = 0; i < numBlocks; i++) { + const tx = await makeBloatedProcessedTx(i + 1); + const emptyTx = await makeEmptyProcessedTx(); + tx.data.constants.historicalHeader = header; + emptyTx.data.constants.historicalHeader = header; + + const blockNum = i + 1000; + + const globals = makeGlobals(blockNum); + + // This will need to be a 2 tx block + const blockTicket = await builder.startNewBlock(2, globals, [], emptyTx); + + await builder.addNewTx(tx); + + // we need to complete the block as we have not added a full set of txs + await builder.setBlockCompleted(); + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + const block = (result as ProvingSuccess).block; + expect(block.number).toEqual(blockNum); + header = block.header; + + await builderDb.commit(); + } + }, 60_000); + it('builds a mixed L2 block', async () => { const txs = await Promise.all([ makeBloatedProcessedTx(1), @@ -500,47 +544,116 @@ describe('prover/tx-prover', () => { expect((result as ProvingSuccess).block.number).toEqual(blockNumber); }, 200_000); - // it('cancels current blocks and switches to new ones', async () => { - // const txs = await Promise.all([ - // makeBloatedProcessedTx(1), - // makeBloatedProcessedTx(2), - // makeBloatedProcessedTx(3), - // makeBloatedProcessedTx(4), - // ]); + it('cancels current block and switches to new ones', async () => { + const txs1 = await Promise.all([ + makeBloatedProcessedTx(1), + makeBloatedProcessedTx(2), + ]); + + const txs2 = await Promise.all([ + makeBloatedProcessedTx(3), + makeBloatedProcessedTx(4), + ]); + + const globals1: GlobalVariables = makeGlobals(100); + const globals2: GlobalVariables = makeGlobals(101); + + const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); + + const blockTicket1 = await builder.startNewBlock( + 2, + globals1, + l1ToL2Messages, + await makeEmptyProcessedTx(), + ); + + await builder.addNewTx(txs1[0]); + await builder.addNewTx(txs1[1]); + + // Now we cancel the block. The first block will come to a stop as and when current proofs complete + builder.cancelBlock(); + + const result1 = await blockTicket1.provingPromise; + + // in all likelihood, the block will have a failure code as we cancelled it + // however it may have actually completed proving before we cancelled in which case it could be a succes code + if (result1.status === PROVING_STATUS.FAILURE) { + expect((result1 as ProvingFailure).reason).toBe('Proving cancelled'); + } + + builderDb.rollback(); + + const blockTicket2 = await builder.startNewBlock( + 2, + globals2, + l1ToL2Messages, + await makeEmptyProcessedTx(), + ); + + await builder.addNewTx(txs2[0]); + await builder.addNewTx(txs2[1]); + + const result2 = await blockTicket2.provingPromise; + expect(result2.status).toBe(PROVING_STATUS.SUCCESS); + const block = (result2 as ProvingSuccess).block; + expect(block.number).toBe(101); + }, 10000); + + it('automatically cancels an incomplete block when starting a new one', async () => { + const txs1 = await Promise.all([ + makeBloatedProcessedTx(1), + makeBloatedProcessedTx(2), + ]); + + const txs2 = await Promise.all([ + makeBloatedProcessedTx(3), + makeBloatedProcessedTx(4), + ]); + + const globals1: GlobalVariables = makeGlobals(100); + const globals2: GlobalVariables = makeGlobals(101); - // const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); + const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); - // const blockPromise1 = await builder.startNewBlock( - // txs.length, - // globalVariables, - // l1ToL2Messages, - // await makeEmptyProcessedTx(), - // ); + const blockTicket1 = await builder.startNewBlock( + 2, + globals1, + l1ToL2Messages, + await makeEmptyProcessedTx(), + ); - // builder.addNewTx(txs[0]); + await builder.addNewTx(txs1[0]); - // const blockPromise2 = await builder.startNewBlock( - // txs.length, - // globalVariables, - // l1ToL2Messages, - // await makeEmptyProcessedTx(), - // ); + builderDb.rollback(); + + const blockTicket2 = await builder.startNewBlock( + 2, + globals2, + l1ToL2Messages, + await makeEmptyProcessedTx(), + ); - // builder.addNewTx(txs[0]); + await builder.addNewTx(txs2[0]); + await builder.addNewTx(txs2[1]); - // await expect(blockPromise1).rejects.toEqual('Block cancelled'); + const result1 = await blockTicket1.provingPromise; + expect(result1.status).toBe(PROVING_STATUS.FAILURE); + expect((result1 as ProvingFailure).reason).toBe('Proving cancelled'); - // const result = await blockPromise2; - // expect(result.block.number).toEqual(blockNumber); - // }, 200_000); + const result2 = await blockTicket2.provingPromise; + expect(result2.status).toBe(PROVING_STATUS.SUCCESS); + const block = (result2 as ProvingSuccess).block; + expect(block.number).toBe(101); + }, 10000); it('builds an unbalanced L2 block', async () => { const txs = await Promise.all([makeEmptyProcessedTx(), makeEmptyProcessedTx(), makeEmptyProcessedTx()]); const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); + // this needs to be a 4 tx block that will need to be completed const blockTicket = await builder.startNewBlock( - txs.length, + 4, globalVariables, l1ToL2Messages, await makeEmptyProcessedTx(), @@ -550,6 +663,8 @@ describe('prover/tx-prover', () => { await builder.addNewTx(tx); } + await builder.setBlockCompleted(); + const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); expect((result as ProvingSuccess).block.number).toEqual(blockNumber); @@ -570,7 +685,7 @@ describe('prover/tx-prover', () => { } await expect(async () => await builder.addNewTx(await makeEmptyProcessedTx())).rejects.toThrow( - `Rollup already contains 4 transactions`, + 'Rollup not accepting further transactions', ); const result = await blockTicket.provingPromise; @@ -582,13 +697,47 @@ describe('prover/tx-prover', () => { await expect(async () => await builder.addNewTx(await makeEmptyProcessedTx())).rejects.toThrow( `Invalid proving state, call startNewBlock before adding transactions`, ); - }, 30_000); + }, 1000); + + it('throws if completing a block before start', async () => { + await expect(async () => await builder.setBlockCompleted()).rejects.toThrow( + 'Invalid proving state, call startNewBlock before adding transactions or completing the block', + ); + }, 1000); + + it('throws if adding to a cancelled block', async () => { + await builder.startNewBlock(2, globalVariables, [], await makeEmptyProcessedTx()); + + builder.cancelBlock(); + + await expect(async () => await builder.addNewTx(await makeEmptyProcessedTx())).rejects.toThrow( + 'Rollup not accepting further transactions', + ); + + }, 10000); + + it.each([ + [-4], + [0], + [1], + [3], + [8.1], + [7] + ] as const)( + 'fails to start a block with %i transaxctions', + async (blockSize: number) => { + await expect( + async () => await builder.startNewBlock(blockSize, globalVariables, [], await makeEmptyProcessedTx()), + ).rejects.toThrow(`Length of txs for the block should be a power of two and at least two (got ${blockSize})`); + }, + 10000, + ); it('rejects if too many l1 to l2 messages are provided', async () => { // Assemble a fake transaction const l1ToL2Messages = new Array(100).fill(new Fr(0n)); await expect( - async () => await builder.startNewBlock(1, globalVariables, l1ToL2Messages, await makeEmptyProcessedTx()), + async () => await builder.startNewBlock(2, globalVariables, l1ToL2Messages, await makeEmptyProcessedTx()), ).rejects.toThrow('Too many L1 to L2 messages'); }); }); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index d813a9dc0d19..46115dbfea13 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -39,7 +39,7 @@ import { getTreeSnapshot, validateTx, } from './block-building-helpers.js'; -import { MergeRollupInputData, PROVING_JOB_TYPE, ProvingJob, ProvingState } from './proving-state.js'; +import { MergeRollupInputData, ProvingState } from './proving-state.js'; const logger = createDebugLogger('aztec:prover:proving-orchestrator'); @@ -62,6 +62,23 @@ enum PROMISE_RESULT { OPERATIONS, } +/** + * Enums and structs to communicate the type of work required in each request. + */ +export enum PROVING_JOB_TYPE { + STATE_UPDATE, + BASE_ROLLUP, + MERGE_ROLLUP, + ROOT_ROLLUP, + BASE_PARITY, + ROOT_PARITY, +} + +export type ProvingJob = { + type: PROVING_JOB_TYPE; + operation: () => Promise; +}; + /** * The orchestrator, managing the flow of recursive proving operations required to build the rollup proof tree. */ @@ -99,7 +116,7 @@ export class ProvingOrchestrator { /** * Starts off a new block - * @param numTxs - The number of real transactions in the block + * @param numTxs - The total number of transactions in the block. Must be a power of 2 * @param globalVariables - The global variables for the block * @param l1ToL2Messages - The l1 to l2 messages for the block * @param emptyTx - The instance of an empty transaction to be used to pad this block @@ -111,9 +128,12 @@ export class ProvingOrchestrator { l1ToL2Messages: Fr[], emptyTx: ProcessedTx, ): Promise { - if (this.provingState && !this.provingState.isFinished()) { - throw new Error("Can't start a new block until the previous block is finished"); + // Check that the length of the array of txs is a power of two + // See https://graphics.stanford.edu/~seander/bithacks.html#DetermineIfPowerOf2 + if (!Number.isInteger(numTxs) || numTxs < 2 || (numTxs & (numTxs - 1)) !== 0) { + throw new Error(`Length of txs for the block should be a power of two and at least two (got ${numTxs})`); } + this.provingState?.cancel(); logger.info(`Starting new block with ${numTxs} transactions`); // we start the block by enqueueing all of the base parity circuits let baseParityInputs: BaseParityInputs[] = []; @@ -130,8 +150,10 @@ export class ProvingOrchestrator { //TODO:(@PhilWindle) Temporary until we figure out when to perform L1 to L2 insertions to make state consistency easier. await Promise.resolve(); + let provingState: ProvingState | undefined = undefined; + const promise = new Promise((resolve, reject) => { - this.provingState = new ProvingState( + provingState = new ProvingState( numTxs, resolve, reject, @@ -143,11 +165,13 @@ export class ProvingOrchestrator { }).catch((reason: string) => ({ status: PROVING_STATUS.FAILURE, reason } as const)); for (let i = 0; i < baseParityInputs.length; i++) { - this.enqueueJob(this.provingState!.Id, PROVING_JOB_TYPE.BASE_PARITY, () => - this.runBaseParityCircuit(baseParityInputs[i], i, this.provingState!.Id), + this.enqueueJob(provingState, PROVING_JOB_TYPE.BASE_PARITY, () => + this.runBaseParityCircuit(provingState, baseParityInputs[i], i), ); } + this.provingState = provingState; + const ticket: ProvingTicket = { provingPromise: promise, }; @@ -163,44 +187,50 @@ export class ProvingOrchestrator { throw new Error(`Invalid proving state, call startNewBlock before adding transactions`); } - if (this.provingState.numTxs === this.provingState.transactionsReceived) { - throw new Error(`Rollup already contains ${this.provingState.transactionsReceived} transactions`); + if (!this.provingState.isAcceptingTransactions()) { + throw new Error(`Rollup not accepting further transactions`); } validateTx(tx); - logger.info(`Received transaction :${tx.hash}`); + logger.info(`Received transaction: ${tx.hash}`); // We start the transaction by enqueueing the state updates - const txIndex = this.provingState!.addNewTx(tx); + const txIndex = this.provingState.addNewTx(tx); // we start this transaction off by performing it's tree insertions and - await this.prepareBaseRollupInputs(BigInt(txIndex), tx, this.provingState!.globalVariables, this.provingState!.Id); - - if (this.provingState.transactionsReceived === this.provingState.numTxs) { - // we need to pad the rollup with empty transactions - const numPaddingTxs = this.provingState.numPaddingTxs; - for (let i = 0; i < numPaddingTxs; i++) { - const paddingTxIndex = this.provingState.addNewTx(this.provingState.emptyTx); - await this.prepareBaseRollupInputs( - BigInt(paddingTxIndex), - this.provingState!.emptyTx, - this.provingState!.globalVariables, - this.provingState!.Id, - ); - } + await this.prepareBaseRollupInputs(this.provingState, BigInt(txIndex), tx); + } + + public async setBlockCompleted() { + if (!this.provingState) { + throw new Error(`Invalid proving state, call startNewBlock before adding transactions or completing the block`); + } + + // we need to pad the rollup with empty transactions + for (let i = this.provingState.transactionsReceived; i < this.provingState.totalNumTxs; i++) { + const paddingTxIndex = this.provingState.addNewTx(this.provingState.emptyTx); + await this.prepareBaseRollupInputs( + this.provingState, + BigInt(paddingTxIndex), + this.provingState!.emptyTx, + ); } } + public cancelBlock() { + this.provingState?.cancel(); + } + /** * Enqueue a job to be scheduled - * @param stateIdentifier - For state Id verification + * @param provingState - The proving state object being operated on * @param jobType - The type of job to be queued * @param job - The actual job, returns a promise notifying of the job's completion */ - private enqueueJob(stateIdentifier: string, jobType: PROVING_JOB_TYPE, job: () => Promise) { - if (!this.provingState!.verifyState(stateIdentifier)) { - logger(`Discarding job for state ID: ${stateIdentifier}`); + private enqueueJob(provingState: ProvingState | undefined, jobType: PROVING_JOB_TYPE, job: () => Promise) { + if (!provingState?.verifyState()) { + logger(`Not enqueueing job, proving state invalid`); return; } // We use a 'safeJob'. We don't want promise rejections in the proving pool, we want to capture the error here @@ -210,7 +240,7 @@ export class ProvingOrchestrator { await job(); } catch (err) { logger.error(`Error thrown when proving job type ${PROVING_JOB_TYPE[jobType]}: ${err}`); - this.provingState!.reject(`${err}`, stateIdentifier); + provingState!.reject(`${err}`); } }; const provingJob: ProvingJob = { @@ -222,12 +252,15 @@ export class ProvingOrchestrator { // Updates the merkle trees for a transaction. The first enqueued job for a transaction private async prepareBaseRollupInputs( + provingState: ProvingState | undefined, index: bigint, tx: ProcessedTx, - globalVariables: GlobalVariables, - stateIdentifier: string, ) { - const inputs = await buildBaseRollupInput(tx, globalVariables, this.db); + if (!provingState?.verifyState()) { + logger('Not preparing base rollup inputs, state invalid'); + return; + } + const inputs = await buildBaseRollupInput(tx, provingState.globalVariables, this.db); const promises = [MerkleTreeId.NOTE_HASH_TREE, MerkleTreeId.NULLIFIER_TREE, MerkleTreeId.PUBLIC_DATA_TREE].map( async (id: MerkleTreeId) => { return { key: id, value: await getTreeSnapshot(id, this.db) }; @@ -237,18 +270,18 @@ export class ProvingOrchestrator { (await Promise.all(promises)).map(obj => [obj.key, obj.value]), ); - if (!this.provingState?.verifyState(stateIdentifier)) { - logger(`Discarding job for state ID: ${stateIdentifier}`); + if (!provingState?.verifyState()) { + logger(`Discarding proving job, state no longer valid`); return; } - - this.enqueueJob(stateIdentifier, PROVING_JOB_TYPE.BASE_ROLLUP, () => - this.runBaseRollup(index, tx, inputs, treeSnapshots, stateIdentifier), + this.enqueueJob(provingState, PROVING_JOB_TYPE.BASE_ROLLUP, () => + this.runBaseRollup(provingState, index, tx, inputs, treeSnapshots), ); } // Stores the intermediate inputs prepared for a merge proof private storeMergeInputs( + provingState: ProvingState, currentLevel: bigint, currentIndex: bigint, mergeInputs: [BaseOrMergeRollupPublicInputs, Proof], @@ -258,19 +291,23 @@ export class ProvingOrchestrator { const mergeIndex = 2n ** mergeLevel - 1n + indexWithinMergeLevel; const subscript = Number(mergeIndex); const indexWithinMerge = Number(currentIndex & 1n); - const ready = this.provingState!.storeMergeInputs(mergeInputs, indexWithinMerge, subscript); - return { ready, indexWithinMergeLevel, mergeLevel, mergeInputData: this.provingState!.getMergeInputs(subscript) }; + const ready = provingState.storeMergeInputs(mergeInputs, indexWithinMerge, subscript); + return { ready, indexWithinMergeLevel, mergeLevel, mergeInputData: provingState.getMergeInputs(subscript) }; } // Executes the base rollup circuit and stored the output as intermediate state for the parent merge/root circuit // Executes the next level of merge if all inputs are available private async runBaseRollup( + provingState: ProvingState | undefined, index: bigint, tx: ProcessedTx, inputs: BaseRollupInputs, treeSnapshots: Map, - stateIdentifier: string, ) { + if (!provingState?.verifyState()) { + logger('Not running base rollup, state invalid'); + return; + } const [duration, baseRollupOutputs] = await elapsed(() => executeBaseRollupCircuit(tx, inputs, treeSnapshots, this.simulator, this.prover, logger), ); @@ -281,23 +318,27 @@ export class ProvingOrchestrator { inputSize: inputs.toBuffer().length, outputSize: baseRollupOutputs[0].toBuffer().length, } satisfies CircuitSimulationStats); - if (!this.provingState?.verifyState(stateIdentifier)) { - logger(`Discarding job for state ID: ${stateIdentifier}`); + if (!provingState?.verifyState()) { + logger(`Discarding job as state no longer valid`); return; } - const currentLevel = this.provingState!.numMergeLevels + 1n; + const currentLevel = provingState.numMergeLevels + 1n; logger.info(`Completed base rollup at index ${index}, current level ${currentLevel}`); - this.storeAndExecuteNextMergeLevel(currentLevel, index, baseRollupOutputs, stateIdentifier); + this.storeAndExecuteNextMergeLevel(provingState, currentLevel, index, baseRollupOutputs); } // Executes the merge rollup circuit and stored the output as intermediate state for the parent merge/root circuit // Executes the next level of merge if all inputs are available private async runMergeRollup( + provingState: ProvingState | undefined, level: bigint, index: bigint, mergeInputData: MergeRollupInputData, - stateIdentifier: string, ) { + if (!provingState?.verifyState()) { + logger('Not running merge rollup, state invalid'); + return; + } const circuitInputs = createMergeRollupInputs( [mergeInputData.inputs[0]!, mergeInputData.proofs[0]!], [mergeInputData.inputs[1]!, mergeInputData.proofs[1]!], @@ -312,25 +353,29 @@ export class ProvingOrchestrator { inputSize: circuitInputs.toBuffer().length, outputSize: circuitOutputs[0].toBuffer().length, } satisfies CircuitSimulationStats); - if (!this.provingState?.verifyState(stateIdentifier)) { - logger(`Discarding job for state ID: ${stateIdentifier}`); + if (!provingState?.verifyState()) { + logger(`Discarding job as state no longer valid`); return; } logger.info(`Completed merge rollup at level ${level}, index ${index}`); - this.storeAndExecuteNextMergeLevel(level, index, circuitOutputs, stateIdentifier); + this.storeAndExecuteNextMergeLevel(provingState, level, index, circuitOutputs); } // Executes the root rollup circuit private async runRootRollup( - mergeInputData: MergeRollupInputData, - rootParityInput: RootParityInput, - stateIdentifier: string, + provingState: ProvingState | undefined, ) { + if (!provingState?.verifyState()) { + logger('Not running root rollup, state no longer valid'); + return; + } + const mergeInputData = provingState.getMergeInputs(0); + const rootParityInput = provingState.finalRootParityInput!; const [circuitsOutput, proof] = await executeRootRollupCircuit( [mergeInputData.inputs[0]!, mergeInputData.proofs[0]!], [mergeInputData.inputs[1]!, mergeInputData.proofs[1]!], rootParityInput, - this.provingState!.newL1ToL2Messages, + provingState!.newL1ToL2Messages, this.simulator, this.prover, this.db, @@ -338,7 +383,7 @@ export class ProvingOrchestrator { ); logger.info(`Completed root rollup`); // Collect all new nullifiers, commitments, and contracts from all txs in this block - const txEffects: TxEffect[] = this.provingState!.allTxs.map(tx => toTxEffect(tx)); + const txEffects: TxEffect[] = provingState.allTxs.map(tx => toTxEffect(tx)); const blockBody = new Body(txEffects); @@ -363,12 +408,16 @@ export class ProvingOrchestrator { proof, }; logger.info(`Successfully proven block ${l2Block.number}!`); - this.provingState!.resolve(provingResult, stateIdentifier); + provingState.resolve(provingResult); } // Executes the base parity circuit and stores the intermediate state for the root parity circuit // Enqueues the root parity circuit if all inputs are available - private async runBaseParityCircuit(inputs: BaseParityInputs, index: number, stateIdentifier: string) { + private async runBaseParityCircuit(provingState: ProvingState | undefined, inputs: BaseParityInputs, index: number) { + if (!provingState?.verifyState()) { + logger('Not running base parity, state no longer valid'); + return; + } const [duration, circuitOutputs] = await elapsed(() => executeBaseParityCircuit(inputs, this.simulator, this.prover, logger), ); @@ -379,27 +428,32 @@ export class ProvingOrchestrator { inputSize: inputs.toBuffer().length, outputSize: circuitOutputs.toBuffer().length, } satisfies CircuitSimulationStats); - if (!this.provingState?.verifyState(stateIdentifier)) { - logger(`Discarding job for state ID: ${stateIdentifier}`); + + if (!provingState?.verifyState()) { + logger(`Discarding job as state no longer valid`); return; } - this.provingState!.setRootParityInputs(circuitOutputs, index); + provingState.setRootParityInputs(circuitOutputs, index); - if (!this.provingState!.areRootParityInputsReady()) { + if (!provingState.areRootParityInputsReady()) { // not ready to run the root parity circuit yet return; } const rootParityInputs = new RootParityInputs( - this.provingState!.rootParityInput as Tuple, + provingState.rootParityInput as Tuple, ); - this.enqueueJob(stateIdentifier, PROVING_JOB_TYPE.ROOT_PARITY, () => - this.runRootParityCircuit(rootParityInputs, stateIdentifier), + this.enqueueJob(provingState, PROVING_JOB_TYPE.ROOT_PARITY, () => + this.runRootParityCircuit(provingState, rootParityInputs), ); } // Runs the root parity circuit ans stored the outputs // Enqueues the root rollup proof if all inputs are available - private async runRootParityCircuit(inputs: RootParityInputs, stateIdentifier: string) { + private async runRootParityCircuit(provingState: ProvingState | undefined, inputs: RootParityInputs) { + if (!provingState?.verifyState()) { + logger(`Not running root parity circuit as state is no longer valid`); + return; + } const [duration, circuitOutputs] = await elapsed(() => executeRootParityCircuit(inputs, this.simulator, this.prover, logger), ); @@ -410,35 +464,34 @@ export class ProvingOrchestrator { inputSize: inputs.toBuffer().length, outputSize: circuitOutputs.toBuffer().length, } satisfies CircuitSimulationStats); - if (!this.provingState?.verifyState(stateIdentifier)) { - logger(`Discarding job for state ID: ${stateIdentifier}`); + + if (!provingState?.verifyState()) { + logger(`Discarding job as state no longer valid`); return; } - this.provingState!.finalRootParityInput = circuitOutputs; - this.checkAndExecuteRootRollup(stateIdentifier); + provingState!.finalRootParityInput = circuitOutputs; + this.checkAndExecuteRootRollup(provingState); } - private checkAndExecuteRootRollup(stateIdentifier: string) { - if (!this.provingState!.isReadyForRootRollup()) { - logger('Not ready for root'); + private checkAndExecuteRootRollup(provingState: ProvingState | undefined) { + if (!provingState?.isReadyForRootRollup()) { + logger('Not ready for root rollup'); return; } - this.enqueueJob(stateIdentifier, PROVING_JOB_TYPE.ROOT_ROLLUP, () => + this.enqueueJob(provingState, PROVING_JOB_TYPE.ROOT_ROLLUP, () => this.runRootRollup( - this.provingState!.getMergeInputs(0)!, - this.provingState!.finalRootParityInput!, - stateIdentifier, + provingState, ), ); } private storeAndExecuteNextMergeLevel( + provingState: ProvingState, currentLevel: bigint, currentIndex: bigint, mergeInputData: [BaseOrMergeRollupPublicInputs, Proof], - stateIdentifier: string, ) { - const result = this.storeMergeInputs(currentLevel, currentIndex, mergeInputData); + const result = this.storeMergeInputs(provingState, currentLevel, currentIndex, mergeInputData); // Are we ready to execute the next circuit? if (!result.ready) { @@ -446,11 +499,11 @@ export class ProvingOrchestrator { } if (result.mergeLevel === 0n) { - this.checkAndExecuteRootRollup(stateIdentifier); + this.checkAndExecuteRootRollup(provingState); } else { // onto the next merge level - this.enqueueJob(stateIdentifier, PROVING_JOB_TYPE.MERGE_ROLLUP, () => - this.runMergeRollup(result.mergeLevel, result.indexWithinMergeLevel, result.mergeInputData, stateIdentifier), + this.enqueueJob(provingState, PROVING_JOB_TYPE.MERGE_ROLLUP, () => + this.runMergeRollup(provingState, result.mergeLevel, result.indexWithinMergeLevel, result.mergeInputData), ); } } diff --git a/yarn-project/prover-client/src/orchestrator/proving-state.ts b/yarn-project/prover-client/src/orchestrator/proving-state.ts index 232f7ad40e58..ebec8c3df078 100644 --- a/yarn-project/prover-client/src/orchestrator/proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/proving-state.ts @@ -7,45 +7,34 @@ import { Proof, RootParityInput, } from '@aztec/circuits.js'; -import { randomBytes } from '@aztec/foundation/crypto'; import { Tuple } from '@aztec/foundation/serialize'; -/** - * Enums and structs to communicate the type of work required in each request. - */ -export enum PROVING_JOB_TYPE { - STATE_UPDATE, - BASE_ROLLUP, - MERGE_ROLLUP, - ROOT_ROLLUP, - BASE_PARITY, - ROOT_PARITY, -} - -export type ProvingJob = { - type: PROVING_JOB_TYPE; - operation: () => Promise; -}; export type MergeRollupInputData = { inputs: [BaseOrMergeRollupPublicInputs | undefined, BaseOrMergeRollupPublicInputs | undefined]; proofs: [Proof | undefined, Proof | undefined]; }; +enum PROVING_STATE_LIFECYCLE { + PROVING_STATE_CREATED, + PROVING_STATE_FULL, + PROVING_STATE_RESOLVED, + PROVING_STATE_REJECTED, +} + /** * The current state of the proving schedule. Contains the raw inputs (txs) and intermediate state to generate every constituent proof in the tree. * Carries an identifier so we can identify if the proving state is discarded and a new one started. * Captures resolve and reject callbacks to provide a promise base interface to the consumer of our proving. */ export class ProvingState { - private stateIdentifier: string; + private provingStateLifecyle = PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED; private mergeRollupInputs: MergeRollupInputData[] = []; private rootParityInputs: Array = []; private finalRootParityInputs: RootParityInput | undefined; - private finished = false; private txs: ProcessedTx[] = []; constructor( - public readonly numTxs: number, + public readonly totalNumTxs: number, private completionCallback: (result: ProvingResult) => void, private rejectionCallback: (reason: string) => void, public readonly globalVariables: GlobalVariables, @@ -53,7 +42,6 @@ export class ProvingState { numRootParityInputs: number, public readonly emptyTx: ProcessedTx, ) { - this.stateIdentifier = randomBytes(32).toString('hex'); this.rootParityInputs = Array.from({ length: numRootParityInputs }).map(_ => undefined); } @@ -65,22 +53,11 @@ export class ProvingState { return this.baseMergeLevel; } - public get Id() { - return this.stateIdentifier; - } - - public get numPaddingTxs() { - return this.totalNumTxs - this.numTxs; - } - - public get totalNumTxs() { - const realTxs = Math.max(2, this.numTxs); - const pow2Txs = Math.ceil(Math.log2(realTxs)); - return 2 ** pow2Txs; - } - public addNewTx(tx: ProcessedTx) { this.txs.push(tx); + if (this.txs.length === this.totalNumTxs) { + this.provingStateLifecyle = PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL; + } return this.txs.length - 1; } @@ -100,8 +77,12 @@ export class ProvingState { return this.rootParityInputs; } - public verifyState(stateId: string) { - return stateId === this.stateIdentifier && !this.finished; + public verifyState() { + return this.provingStateLifecyle === PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED || this.provingStateLifecyle === PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL; + } + + public isAcceptingTransactions() { + return this.provingStateLifecyle === PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED; } public get allTxs() { @@ -134,16 +115,7 @@ export class ProvingState { } public isReadyForRootRollup() { - if (this.mergeRollupInputs[0] === undefined) { - return false; - } - if (this.mergeRollupInputs[0].inputs.findIndex(p => !p) !== -1) { - return false; - } - if (this.finalRootParityInput === undefined) { - return false; - } - return true; + return !(this.mergeRollupInputs[0] === undefined || this.finalRootParityInput === undefined || this.mergeRollupInputs[0].inputs.findIndex(p => !p) !== -1); } public setRootParityInputs(inputs: RootParityInput, index: number) { @@ -154,29 +126,23 @@ export class ProvingState { return this.rootParityInputs.findIndex(p => !p) === -1; } - public reject(reason: string, stateIdentifier: string) { - if (!this.verifyState(stateIdentifier)) { - return; - } - if (this.finished) { + public cancel() { + this.reject('Proving cancelled'); + } + + public reject(reason: string) { + if (!this.verifyState()) { return; } - this.finished = true; + this.provingStateLifecyle = PROVING_STATE_LIFECYCLE.PROVING_STATE_REJECTED; this.rejectionCallback(reason); } - public resolve(result: ProvingResult, stateIdentifier: string) { - if (!this.verifyState(stateIdentifier)) { + public resolve(result: ProvingResult) { + if (!this.verifyState()) { return; } - if (this.finished) { - return; - } - this.finished = true; + this.provingStateLifecyle = PROVING_STATE_LIFECYCLE.PROVING_STATE_RESOLVED; this.completionCallback(result); } - - public isFinished() { - return this.finished; - } } From bd10688b59bb3943297467471f50bdf9fee149ed Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 1 Apr 2024 15:09:10 +0100 Subject: [PATCH 02/24] WIP --- .../src/interfaces/block-prover.ts | 11 +- .../src/integration_l1_publisher.test.ts | 6 +- .../prover-client/src/dummy-prover.ts | 15 ++- .../orchestrator/block-building-helpers.ts | 35 ++---- .../src/orchestrator/orchestrator.test.ts | 73 ++++++++++--- .../src/orchestrator/orchestrator.ts | 102 +++++++++++++----- .../src/orchestrator/proving-state.ts | 10 +- .../prover-client/src/tx-prover/tx-prover.ts | 16 ++- .../src/sequencer/sequencer.test.ts | 14 +-- .../src/sequencer/sequencer.ts | 8 +- 10 files changed, 201 insertions(+), 89 deletions(-) diff --git a/yarn-project/circuit-types/src/interfaces/block-prover.ts b/yarn-project/circuit-types/src/interfaces/block-prover.ts index 9e02a117b22b..fb60de25e5de 100644 --- a/yarn-project/circuit-types/src/interfaces/block-prover.ts +++ b/yarn-project/circuit-types/src/interfaces/block-prover.ts @@ -10,8 +10,6 @@ export enum PROVING_STATUS { export type ProvingSuccess = { status: PROVING_STATUS.SUCCESS; - block: L2Block; - proof: Proof; }; export type ProvingFailure = { @@ -25,6 +23,11 @@ export type ProvingTicket = { provingPromise: Promise; }; +export type BlockResult = { + block: L2Block, + proof: Proof; +} + /** * The interface to the block prover. * Provides the ability to generate proofs and build rollups. @@ -38,4 +41,8 @@ export interface BlockProver { ): Promise; addNewTx(tx: ProcessedTx): Promise; + + cancelBlock(): void; + + finaliseBlock(): Promise; } diff --git a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts index 050cf0da6bf7..c7b86af2f3ed 100644 --- a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts @@ -381,7 +381,8 @@ describe('L1Publisher integration', () => { ); const ticket = await buildBlock(globalVariables, txs, currentL1ToL2Messages, makeEmptyProcessedTx()); const result = await ticket.provingPromise; - const block = (result as ProvingSuccess).block; + const blockResult = await builder.finaliseBlock(); + const block = blockResult.block; prevHeader = block.header; const newL2ToL1MsgsArray = block.body.txEffects.flatMap(txEffect => txEffect.l2ToL1Msgs); @@ -471,7 +472,8 @@ describe('L1Publisher integration', () => { ); const blockTicket = await buildBlock(globalVariables, txs, l1ToL2Messages, makeEmptyProcessedTx()); const result = await blockTicket.provingPromise; - const block = (result as ProvingSuccess).block; + const blockResult = await builder.finaliseBlock(); + const block = blockResult.block; prevHeader = block.header; writeJson(`empty_block_${i}`, block, [], AztecAddress.ZERO, deployerAccount.address); diff --git a/yarn-project/prover-client/src/dummy-prover.ts b/yarn-project/prover-client/src/dummy-prover.ts index d2c1f4842e40..ba8ea7c5ae4b 100644 --- a/yarn-project/prover-client/src/dummy-prover.ts +++ b/yarn-project/prover-client/src/dummy-prover.ts @@ -1,4 +1,5 @@ import { + BlockResult, L2Block, PROVING_STATUS, ProcessedTx, @@ -10,6 +11,8 @@ import { GlobalVariables, makeEmptyProof } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; export class DummyProver implements ProverClient { + + public start(): Promise { return Promise.resolve(); } @@ -30,8 +33,6 @@ export class DummyProver implements ProverClient { ): Promise { const result: ProvingSuccess = { status: PROVING_STATUS.SUCCESS, - proof: makeEmptyProof(), - block: L2Block.empty(), }; const ticket: ProvingTicket = { provingPromise: Promise.resolve(result), @@ -42,4 +43,14 @@ export class DummyProver implements ProverClient { addNewTx(_tx: ProcessedTx): Promise { return Promise.resolve(); } + + cancelBlock(): void { + } + + finaliseBlock(): Promise { + return Promise.resolve({ + block: L2Block.empty(), + proof: makeEmptyProof(), + }); + } } diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index 5156edd63015..e048854d8dd4 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -205,29 +205,21 @@ export async function executeRootRollupCircuit( right: [BaseOrMergeRollupPublicInputs, Proof], l1ToL2Roots: RootParityInput, newL1ToL2Messages: Tuple, + messageTreeSnapshot: AppendOnlyTreeSnapshot, + messageTreeRootSiblingPath: Tuple, simulator: RollupSimulator, prover: RollupProver, db: MerkleTreeOperations, logger?: DebugLogger, ): Promise<[RootRollupPublicInputs, Proof]> { logger?.debug(`Running root rollup circuit`); - const rootInput = await getRootRollupInput(...left, ...right, l1ToL2Roots, newL1ToL2Messages, db); - - // Update the local trees to include the new l1 to l2 messages - await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, newL1ToL2Messages); + const rootInput = await getRootRollupInput(...left, ...right, l1ToL2Roots, newL1ToL2Messages, messageTreeSnapshot, messageTreeRootSiblingPath, db); // Simulate and get proof for the root circuit const rootOutput = await simulator.rootRollupCircuit(rootInput); const rootProof = await prover.getRootRollupProof(rootInput, rootOutput); - //TODO(@PhilWindle) Move this to orchestrator to ensure that we are still on the same block - // Update the archive with the latest block header - logger?.debug(`Updating and validating root trees`); - await db.updateArchive(rootOutput.header); - - await validateRootOutput(rootOutput, db); - return [rootOutput, rootProof]; } @@ -264,6 +256,8 @@ export async function getRootRollupInput( rollupProofRight: Proof, l1ToL2Roots: RootParityInput, newL1ToL2Messages: Tuple, + messageTreeSnapshot: AppendOnlyTreeSnapshot, + messageTreeRootSiblingPath: Tuple, db: MerkleTreeOperations, ) { const vks = getVerificationKeys(); @@ -279,21 +273,6 @@ export async function getRootRollupInput( return path.toFields(); }; - const newL1ToL2MessageTreeRootSiblingPathArray = await getSubtreeSiblingPath( - MerkleTreeId.L1_TO_L2_MESSAGE_TREE, - L1_TO_L2_MSG_SUBTREE_HEIGHT, - db, - ); - - const newL1ToL2MessageTreeRootSiblingPath = makeTuple( - L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, - i => (i < newL1ToL2MessageTreeRootSiblingPathArray.length ? newL1ToL2MessageTreeRootSiblingPathArray[i] : Fr.ZERO), - 0, - ); - - // Get tree snapshots - const startL1ToL2MessageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db); - // Get blocks tree const startArchiveSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db); const newArchiveSiblingPathArray = await getRootTreeSiblingPath(MerkleTreeId.ARCHIVE); @@ -308,8 +287,8 @@ export async function getRootRollupInput( previousRollupData, l1ToL2Roots, newL1ToL2Messages, - newL1ToL2MessageTreeRootSiblingPath, - startL1ToL2MessageTreeSnapshot, + newL1ToL2MessageTreeRootSiblingPath: messageTreeRootSiblingPath, + startL1ToL2MessageTreeSnapshot: messageTreeSnapshot, startArchiveSnapshot, newArchiveSiblingPath, }); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts index 409b572bd474..aeb69b08d334 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts @@ -407,7 +407,9 @@ describe('prover/tx-prover', () => { const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); - expect((result as ProvingSuccess).block.number).toEqual(blockNumber); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.l2Block.number).toEqual(blockNumber); await updateExpectedTreesFromTxs(txs); const noteHashTreeAfter = await builderDb.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); @@ -438,7 +440,9 @@ describe('prover/tx-prover', () => { const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); - expect((result as ProvingSuccess).block.number).toEqual(blockNumber); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.l2Block.number).toEqual(blockNumber); }, 30_000); it('builds a block with 1 transaction', async () => { @@ -456,7 +460,9 @@ describe('prover/tx-prover', () => { const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); - expect((result as ProvingSuccess).block.number).toEqual(blockNumber); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.l2Block.number).toEqual(blockNumber); }, 30_000); it('builds multiple blocks in sequence', async () => { @@ -483,9 +489,10 @@ describe('prover/tx-prover', () => { const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); - const block = (result as ProvingSuccess).block; - expect(block.number).toEqual(blockNum); - header = block.header; + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.l2Block.number).toEqual(blockNum); + header = finalisedBlock.l2Block.header; await builderDb.commit(); } @@ -514,7 +521,9 @@ describe('prover/tx-prover', () => { const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); - expect((result as ProvingSuccess).block.number).toEqual(blockNumber); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.l2Block.number).toEqual(blockNumber); }, 200_000); it('builds a block concurrently with transactions', async () => { @@ -541,7 +550,9 @@ describe('prover/tx-prover', () => { const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); - expect((result as ProvingSuccess).block.number).toEqual(blockNumber); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.l2Block.number).toEqual(blockNumber); }, 200_000); it('cancels current block and switches to new ones', async () => { @@ -595,8 +606,9 @@ describe('prover/tx-prover', () => { const result2 = await blockTicket2.provingPromise; expect(result2.status).toBe(PROVING_STATUS.SUCCESS); - const block = (result2 as ProvingSuccess).block; - expect(block.number).toBe(101); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.l2Block.number).toEqual(101); }, 10000); it('automatically cancels an incomplete block when starting a new one', async () => { @@ -642,8 +654,9 @@ describe('prover/tx-prover', () => { const result2 = await blockTicket2.provingPromise; expect(result2.status).toBe(PROVING_STATUS.SUCCESS); - const block = (result2 as ProvingSuccess).block; - expect(block.number).toBe(101); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.l2Block.number).toEqual(101); }, 10000); it('builds an unbalanced L2 block', async () => { @@ -667,7 +680,9 @@ describe('prover/tx-prover', () => { const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); - expect((result as ProvingSuccess).block.number).toEqual(blockNumber); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.l2Block.number).toEqual(blockNumber); }, 200_000); it('throws if adding too many transactions', async () => { @@ -690,7 +705,9 @@ describe('prover/tx-prover', () => { const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); - expect((result as ProvingSuccess).block.number).toEqual(blockNumber); + const finalisedBlock = await builder.finaliseBlock(); + + expect(finalisedBlock.l2Block.number).toEqual(blockNumber); }, 30_000); it('throws if adding a transaction before start', async () => { @@ -705,6 +722,34 @@ describe('prover/tx-prover', () => { ); }, 1000); + it('throws if finalising an incompletre block', async () => { + await expect(async () => await builder.finaliseBlock()).rejects.toThrow( + 'Invalid proving state, a block must be proven before it can be finalised', + ); + }, 1000); + + it('throws if finalising an already finalised block', async () => { + + const txs = await Promise.all([ + makeEmptyProcessedTx(), + makeEmptyProcessedTx(), + ]); + + const blockTicket = await builder.startNewBlock(txs.length, globalVariables, [], await makeEmptyProcessedTx()); + + for (const tx of txs) { + await builder.addNewTx(tx); + } + + const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); + const finalisedBlock = await builder.finaliseBlock(); + expect(finalisedBlock.l2Block.number).toEqual(blockNumber); + await expect(async () => await builder.finaliseBlock()).rejects.toThrow( + 'Block already finalised', + ); + }, 20000); + it('throws if adding to a cancelled block', async () => { await builder.startNewBlock(2, globalVariables, [], await makeEmptyProcessedTx()); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index 46115dbfea13..8c157a9d523f 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -1,5 +1,5 @@ import { Body, L2Block, MerkleTreeId, ProcessedTx, TxEffect, toTxEffect } from '@aztec/circuit-types'; -import { PROVING_STATUS, ProvingResult, ProvingTicket } from '@aztec/circuit-types/interfaces'; +import { PROVING_STATUS, ProverClient, ProvingResult, ProvingTicket } from '@aztec/circuit-types/interfaces'; import { CircuitSimulationStats } from '@aztec/circuit-types/stats'; import { AppendOnlyTreeSnapshot, @@ -8,6 +8,8 @@ import { BaseRollupInputs, Fr, GlobalVariables, + L1_TO_L2_MSG_SUBTREE_HEIGHT, + L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NUM_BASE_PARITY_PER_ROOT_PARITY, Proof, @@ -36,10 +38,13 @@ import { executeMergeRollupCircuit, executeRootParityCircuit, executeRootRollupCircuit, + getSubtreeSiblingPath, getTreeSnapshot, + validateRootOutput, validateTx, } from './block-building-helpers.js'; import { MergeRollupInputData, ProvingState } from './proving-state.js'; +import { makeTuple } from '@aztec/foundation/array'; const logger = createDebugLogger('aztec:prover:proving-orchestrator'); @@ -51,7 +56,7 @@ const logger = createDebugLogger('aztec:prover:proving-orchestrator'); * 4. Once a transaction is proven, it will be incorporated into a merge proof * 5. Merge proofs are produced at each level of the tree until the root proof is produced * - * The proving implementation is determined by the provided prover implementation. This could be for example a local prover or a remote prover pool. + * The proving implementation is determined by the provided prover. This could be for example a local prover or a remote prover pool. */ const SLEEP_TIME = 50; @@ -106,6 +111,7 @@ export class ProvingOrchestrator { public start() { this.jobProcessPromise = this.processJobQueue(); + return Promise.resolve(); } public async stop() { @@ -147,8 +153,22 @@ export class ProvingOrchestrator { BaseParityInputs.fromSlice(l1ToL2MessagesPadded, i), ); - //TODO:(@PhilWindle) Temporary until we figure out when to perform L1 to L2 insertions to make state consistency easier. - await Promise.resolve(); + const messageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, this.db); + + const newL1ToL2MessageTreeRootSiblingPathArray = await getSubtreeSiblingPath( + MerkleTreeId.L1_TO_L2_MESSAGE_TREE, + L1_TO_L2_MSG_SUBTREE_HEIGHT, + this.db, + ); + + const newL1ToL2MessageTreeRootSiblingPath = makeTuple( + L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, + i => (i < newL1ToL2MessageTreeRootSiblingPathArray.length ? newL1ToL2MessageTreeRootSiblingPathArray[i] : Fr.ZERO), + 0, + ); + + // Update the local trees to include the new l1 to l2 messages + await this.db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded); let provingState: ProvingState | undefined = undefined; @@ -161,6 +181,8 @@ export class ProvingOrchestrator { l1ToL2MessagesPadded, baseParityInputs.length, emptyTx, + messageTreeSnapshot, + newL1ToL2MessageTreeRootSiblingPath, ); }).catch((reason: string) => ({ status: PROVING_STATUS.FAILURE, reason } as const)); @@ -222,6 +244,51 @@ export class ProvingOrchestrator { this.provingState?.cancel(); } + public async finaliseBlock() { + if (!this.provingState || !this.provingState.rootRollupPublicInputs || !this.provingState.finalProof) { + throw new Error(`Invalid proving state, a block must be proven before it can be finalised`); + } + if (this.provingState.block) { + throw new Error('Block already finalised'); + } + + const rootRollupOutputs = this.provingState.rootRollupPublicInputs; + + logger?.debug(`Updating and validating root trees`); + await this.db.updateArchive(rootRollupOutputs.header); + + await validateRootOutput(rootRollupOutputs, this.db); + + // Collect all new nullifiers, commitments, and contracts from all txs in this block + const txEffects: TxEffect[] = this.provingState.allTxs.map(tx => toTxEffect(tx)); + + const blockBody = new Body(txEffects); + + const l2Block = L2Block.fromFields({ + archive: rootRollupOutputs.archive, + header: rootRollupOutputs.header, + body: blockBody, + }); + + if (!l2Block.body.getTxsEffectsHash().equals(rootRollupOutputs.header.contentCommitment.txsEffectsHash)) { + logger(inspect(blockBody)); + throw new Error( + `Txs effects hash mismatch, ${l2Block.body + .getTxsEffectsHash() + .toString('hex')} == ${rootRollupOutputs.header.contentCommitment.txsEffectsHash.toString('hex')} `, + ); + } + + logger.info(`Successfully proven block ${l2Block.number}!`); + + this.provingState.block = l2Block; + + return { + l2Block, + proof: this.provingState.finalProof, + } + } + /** * Enqueue a job to be scheduled * @param provingState - The proving state object being operated on @@ -375,39 +442,22 @@ export class ProvingOrchestrator { [mergeInputData.inputs[0]!, mergeInputData.proofs[0]!], [mergeInputData.inputs[1]!, mergeInputData.proofs[1]!], rootParityInput, - provingState!.newL1ToL2Messages, + provingState.newL1ToL2Messages, + provingState.messageTreeSnapshot, + provingState.messageTreeRootSiblingPath, this.simulator, this.prover, this.db, logger, ); logger.info(`Completed root rollup`); - // Collect all new nullifiers, commitments, and contracts from all txs in this block - const txEffects: TxEffect[] = provingState.allTxs.map(tx => toTxEffect(tx)); - const blockBody = new Body(txEffects); - - const l2Block = L2Block.fromFields({ - archive: circuitsOutput.archive, - header: circuitsOutput.header, - body: blockBody, - }); - - if (!l2Block.body.getTxsEffectsHash().equals(circuitsOutput.header.contentCommitment.txsEffectsHash)) { - logger(inspect(blockBody)); - throw new Error( - `Txs effects hash mismatch, ${l2Block.body - .getTxsEffectsHash() - .toString('hex')} == ${circuitsOutput.header.contentCommitment.txsEffectsHash.toString('hex')} `, - ); - } + provingState.rootRollupPublicInputs = circuitsOutput; + provingState.finalProof = proof; const provingResult: ProvingResult = { status: PROVING_STATUS.SUCCESS, - block: l2Block, - proof, }; - logger.info(`Successfully proven block ${l2Block.number}!`); provingState.resolve(provingResult); } diff --git a/yarn-project/prover-client/src/orchestrator/proving-state.ts b/yarn-project/prover-client/src/orchestrator/proving-state.ts index ebec8c3df078..7d0f42d5115c 100644 --- a/yarn-project/prover-client/src/orchestrator/proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/proving-state.ts @@ -1,11 +1,14 @@ -import { ProcessedTx, ProvingResult } from '@aztec/circuit-types'; +import { L2Block, ProcessedTx, ProvingResult } from '@aztec/circuit-types'; import { + AppendOnlyTreeSnapshot, BaseOrMergeRollupPublicInputs, Fr, GlobalVariables, + L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, Proof, RootParityInput, + RootRollupPublicInputs, } from '@aztec/circuits.js'; import { Tuple } from '@aztec/foundation/serialize'; @@ -32,6 +35,9 @@ export class ProvingState { private mergeRollupInputs: MergeRollupInputData[] = []; private rootParityInputs: Array = []; private finalRootParityInputs: RootParityInput | undefined; + public rootRollupPublicInputs: RootRollupPublicInputs | undefined; + public finalProof: Proof | undefined; + public block: L2Block | undefined; private txs: ProcessedTx[] = []; constructor( public readonly totalNumTxs: number, @@ -41,6 +47,8 @@ export class ProvingState { public readonly newL1ToL2Messages: Tuple, numRootParityInputs: number, public readonly emptyTx: ProcessedTx, + public readonly messageTreeSnapshot: AppendOnlyTreeSnapshot, + public readonly messageTreeRootSiblingPath: Tuple, ) { this.rootParityInputs = Array.from({ length: numRootParityInputs }).map(_ => undefined); } diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index 782b65d14c24..02f2ed8b1305 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -1,5 +1,5 @@ import { ProcessedTx } from '@aztec/circuit-types'; -import { ProverClient, ProvingTicket } from '@aztec/circuit-types/interfaces'; +import { BlockResult, ProverClient, ProvingTicket } from '@aztec/circuit-types/interfaces'; import { Fr, GlobalVariables } from '@aztec/circuits.js'; import { SimulationProvider } from '@aztec/simulator'; import { WorldStateSynchronizer } from '@aztec/world-state'; @@ -15,7 +15,7 @@ import { EmptyRollupProver } from '../prover/empty.js'; export class TxProver implements ProverClient { private orchestrator: ProvingOrchestrator; constructor( - worldStateSynchronizer: WorldStateSynchronizer, + private worldStateSynchronizer: WorldStateSynchronizer, simulationProvider: SimulationProvider, protected vks: VerificationKeys, ) { @@ -27,6 +27,7 @@ export class TxProver implements ProverClient { ); } + /** * Starts the prover instance */ @@ -58,16 +59,25 @@ export class TxProver implements ProverClient { return prover; } - public startNewBlock( + public async startNewBlock( numTxs: number, globalVariables: GlobalVariables, newL1ToL2Messages: Fr[], emptyTx: ProcessedTx, ): Promise { + const previousBlockNumber = globalVariables.blockNumber.toNumber() - 1; + await this.worldStateSynchronizer.syncImmediate(previousBlockNumber); return this.orchestrator.startNewBlock(numTxs, globalVariables, newL1ToL2Messages, emptyTx); } public addNewTx(tx: ProcessedTx): Promise { return this.orchestrator.addNewTx(tx); } + + public cancelBlock(): void { + this.orchestrator.cancelBlock(); + } + public finaliseBlock(): Promise { + return this.finaliseBlock(); + } } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index ed46187cd94c..da596b4c16c5 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -119,8 +119,6 @@ describe('sequencer', () => { const proof = makeEmptyProof(); const result: ProvingSuccess = { status: PROVING_STATUS.SUCCESS, - proof, - block, }; const ticket: ProvingTicket = { provingPromise: Promise.resolve(result), @@ -128,6 +126,7 @@ describe('sequencer', () => { p2p.getTxs.mockResolvedValueOnce([tx]); proverClient.startNewBlock.mockResolvedValueOnce(ticket); + proverClient.finaliseBlock.mockResolvedValue({block, proof}); publisher.processL2Block.mockResolvedValueOnce(true); globalVariableBuilder.buildGlobalVariables.mockResolvedValueOnce( new GlobalVariables(chainId, version, new Fr(lastBlockNumber + 1), Fr.ZERO, coinbase, feeRecipient), @@ -137,7 +136,7 @@ describe('sequencer', () => { await sequencer.work(); expect(proverClient.startNewBlock).toHaveBeenCalledWith( - 1, + 2, new GlobalVariables(chainId, version, new Fr(lastBlockNumber + 1), Fr.ZERO, coinbase, feeRecipient), Array(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP).fill(new Fr(0n)), publicProcessor.makeEmptyProcessedTx(), @@ -158,8 +157,6 @@ describe('sequencer', () => { const proof = makeEmptyProof(); const result: ProvingSuccess = { status: PROVING_STATUS.SUCCESS, - proof, - block, }; const ticket: ProvingTicket = { provingPromise: Promise.resolve(result), @@ -167,6 +164,7 @@ describe('sequencer', () => { p2p.getTxs.mockResolvedValueOnce(txs); proverClient.startNewBlock.mockResolvedValueOnce(ticket); + proverClient.finaliseBlock.mockResolvedValue({block, proof}); publisher.processL2Block.mockResolvedValueOnce(true); globalVariableBuilder.buildGlobalVariables.mockResolvedValueOnce( new GlobalVariables(chainId, version, new Fr(lastBlockNumber + 1), Fr.ZERO, coinbase, feeRecipient), @@ -207,8 +205,6 @@ describe('sequencer', () => { const proof = makeEmptyProof(); const result: ProvingSuccess = { status: PROVING_STATUS.SUCCESS, - proof, - block, }; const ticket: ProvingTicket = { provingPromise: Promise.resolve(result), @@ -216,6 +212,7 @@ describe('sequencer', () => { p2p.getTxs.mockResolvedValueOnce(txs); proverClient.startNewBlock.mockResolvedValueOnce(ticket); + proverClient.finaliseBlock.mockResolvedValue({block, proof}); publisher.processL2Block.mockResolvedValueOnce(true); globalVariableBuilder.buildGlobalVariables.mockResolvedValueOnce( new GlobalVariables(chainId, version, new Fr(lastBlockNumber + 1), Fr.ZERO, coinbase, feeRecipient), @@ -246,8 +243,6 @@ describe('sequencer', () => { const proof = makeEmptyProof(); const result: ProvingSuccess = { status: PROVING_STATUS.SUCCESS, - proof, - block, }; const ticket: ProvingTicket = { provingPromise: Promise.resolve(result), @@ -255,6 +250,7 @@ describe('sequencer', () => { p2p.getTxs.mockResolvedValueOnce([tx]); proverClient.startNewBlock.mockResolvedValueOnce(ticket); + proverClient.finaliseBlock.mockResolvedValue({block, proof}); publisher.processL2Block.mockResolvedValueOnce(true); globalVariableBuilder.buildGlobalVariables.mockResolvedValueOnce( new GlobalVariables(chainId, version, new Fr(lastBlockNumber + 1), Fr.ZERO, coinbase, feeRecipient), diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 4823f48d3fa6..2ef996d6c635 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -303,7 +303,10 @@ export class Sequencer { emptyTx: ProcessedTx, globalVariables: GlobalVariables, ) { - const blockTicket = await this.prover.startNewBlock(txs.length, globalVariables, l1ToL2Messages, emptyTx); + const numRealTxs = txs.length; + const pow2 = Math.log2(numRealTxs); + const totalTxs = 2 ** Math.ceil(pow2); + const blockTicket = await this.prover.startNewBlock(Math.max(totalTxs, 2), globalVariables, l1ToL2Messages, emptyTx); for (const tx of txs) { await this.prover.addNewTx(tx); @@ -313,7 +316,8 @@ export class Sequencer { if (result.status === PROVING_STATUS.FAILURE) { throw new Error(`Block proving failed, reason: ${result.reason}`); } - return result.block; + const blockResult = await this.prover.finaliseBlock(); + return blockResult.block; } get coinbase(): EthAddress { From c78303ce6394b3b386e618745c3dd91d64c0a228 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 2 Apr 2024 15:34:26 +0100 Subject: [PATCH 03/24] Formatting --- yarn-project/circuit-types/src/interfaces/block-prover.ts | 4 ++-- .../end-to-end/src/integration_l1_publisher.test.ts | 6 ++++-- .../src/orchestrator/block-building-helpers.ts | 3 +-- .../prover-client/src/orchestrator/orchestrator.test.ts | 7 +++---- .../prover-client/src/orchestrator/orchestrator.ts | 4 ++-- .../sequencer-client/src/sequencer/sequencer.test.ts | 8 ++++---- yarn-project/sequencer-client/src/sequencer/sequencer.ts | 7 ++++++- 7 files changed, 22 insertions(+), 17 deletions(-) diff --git a/yarn-project/circuit-types/src/interfaces/block-prover.ts b/yarn-project/circuit-types/src/interfaces/block-prover.ts index fb60de25e5de..4e9c14805c06 100644 --- a/yarn-project/circuit-types/src/interfaces/block-prover.ts +++ b/yarn-project/circuit-types/src/interfaces/block-prover.ts @@ -24,9 +24,9 @@ export type ProvingTicket = { }; export type BlockResult = { - block: L2Block, + block: L2Block; proof: Proof; -} +}; /** * The interface to the block prover. diff --git a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts index c7b86af2f3ed..994b95f324fa 100644 --- a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts @@ -3,10 +3,10 @@ import { getConfigEnvVars } from '@aztec/aztec-node'; import { AztecAddress, Body, Fr, GlobalVariables, L2Actor, L2Block, createDebugLogger, mockTx } from '@aztec/aztec.js'; // eslint-disable-next-line no-restricted-imports import { + PROVING_STATUS, ProcessedTx, - ProvingSuccess, makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, - makeProcessedTx, + makeProcessedTx } from '@aztec/circuit-types'; import { EthAddress, @@ -381,6 +381,7 @@ describe('L1Publisher integration', () => { ); const ticket = await buildBlock(globalVariables, txs, currentL1ToL2Messages, makeEmptyProcessedTx()); const result = await ticket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); const blockResult = await builder.finaliseBlock(); const block = blockResult.block; prevHeader = block.header; @@ -472,6 +473,7 @@ describe('L1Publisher integration', () => { ); const blockTicket = await buildBlock(globalVariables, txs, l1ToL2Messages, makeEmptyProcessedTx()); const result = await blockTicket.provingPromise; + expect(result.status).toBe(PROVING_STATUS.SUCCESS); const blockResult = await builder.finaliseBlock(); const block = blockResult.block; prevHeader = block.header; diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index e048854d8dd4..c880605474d1 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -8,7 +8,6 @@ import { ConstantRollupData, Fr, GlobalVariables, - L1_TO_L2_MSG_SUBTREE_HEIGHT, L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, MAX_NEW_NULLIFIERS_PER_TX, MAX_PUBLIC_DATA_READS_PER_TX, @@ -41,7 +40,7 @@ import { StateDiffHints, StateReference, VK_TREE_HEIGHT, - VerificationKey, + VerificationKey } from '@aztec/circuits.js'; import { assertPermutation, makeTuple } from '@aztec/foundation/array'; import { DebugLogger } from '@aztec/foundation/log'; diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts index aeb69b08d334..3095b945fdeb 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts @@ -3,10 +3,9 @@ import { PROVING_STATUS, ProcessedTx, ProvingFailure, - ProvingSuccess, makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, makeProcessedTx, - mockTx, + mockTx } from '@aztec/circuit-types'; import { AztecAddress, @@ -52,13 +51,13 @@ import { WASMSimulator } from '@aztec/simulator'; import { MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { MockProxy, mock } from 'jest-mock-extended'; -import { type MemDown, default as memdown } from 'memdown'; +import { default as memdown, type MemDown } from 'memdown'; +import { createDebugLogger } from '@aztec/foundation/log'; import { getVerificationKeys } from '../mocks/verification_keys.js'; import { RollupProver } from '../prover/index.js'; import { RollupSimulator } from '../simulator/rollup.js'; import { ProvingOrchestrator } from './orchestrator.js'; -import { createDebugLogger } from '@aztec/foundation/log'; const logger = createDebugLogger('aztec:orchestrator-test'); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index 8c157a9d523f..b6b354c2df9e 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -1,5 +1,5 @@ import { Body, L2Block, MerkleTreeId, ProcessedTx, TxEffect, toTxEffect } from '@aztec/circuit-types'; -import { PROVING_STATUS, ProverClient, ProvingResult, ProvingTicket } from '@aztec/circuit-types/interfaces'; +import { PROVING_STATUS, ProvingResult, ProvingTicket } from '@aztec/circuit-types/interfaces'; import { CircuitSimulationStats } from '@aztec/circuit-types/stats'; import { AppendOnlyTreeSnapshot, @@ -27,6 +27,7 @@ import { MerkleTreeOperations } from '@aztec/world-state'; import { inspect } from 'util'; +import { makeTuple } from '@aztec/foundation/array'; import { VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; import { RollupProver } from '../prover/index.js'; import { RealRollupCircuitSimulator, RollupSimulator } from '../simulator/rollup.js'; @@ -44,7 +45,6 @@ import { validateTx, } from './block-building-helpers.js'; import { MergeRollupInputData, ProvingState } from './proving-state.js'; -import { makeTuple } from '@aztec/foundation/array'; const logger = createDebugLogger('aztec:prover:proving-orchestrator'); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index da596b4c16c5..6e0f270f97af 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -126,7 +126,7 @@ describe('sequencer', () => { p2p.getTxs.mockResolvedValueOnce([tx]); proverClient.startNewBlock.mockResolvedValueOnce(ticket); - proverClient.finaliseBlock.mockResolvedValue({block, proof}); + proverClient.finaliseBlock.mockResolvedValue({ block, proof }); publisher.processL2Block.mockResolvedValueOnce(true); globalVariableBuilder.buildGlobalVariables.mockResolvedValueOnce( new GlobalVariables(chainId, version, new Fr(lastBlockNumber + 1), Fr.ZERO, coinbase, feeRecipient), @@ -164,7 +164,7 @@ describe('sequencer', () => { p2p.getTxs.mockResolvedValueOnce(txs); proverClient.startNewBlock.mockResolvedValueOnce(ticket); - proverClient.finaliseBlock.mockResolvedValue({block, proof}); + proverClient.finaliseBlock.mockResolvedValue({ block, proof }); publisher.processL2Block.mockResolvedValueOnce(true); globalVariableBuilder.buildGlobalVariables.mockResolvedValueOnce( new GlobalVariables(chainId, version, new Fr(lastBlockNumber + 1), Fr.ZERO, coinbase, feeRecipient), @@ -212,7 +212,7 @@ describe('sequencer', () => { p2p.getTxs.mockResolvedValueOnce(txs); proverClient.startNewBlock.mockResolvedValueOnce(ticket); - proverClient.finaliseBlock.mockResolvedValue({block, proof}); + proverClient.finaliseBlock.mockResolvedValue({ block, proof }); publisher.processL2Block.mockResolvedValueOnce(true); globalVariableBuilder.buildGlobalVariables.mockResolvedValueOnce( new GlobalVariables(chainId, version, new Fr(lastBlockNumber + 1), Fr.ZERO, coinbase, feeRecipient), @@ -250,7 +250,7 @@ describe('sequencer', () => { p2p.getTxs.mockResolvedValueOnce([tx]); proverClient.startNewBlock.mockResolvedValueOnce(ticket); - proverClient.finaliseBlock.mockResolvedValue({block, proof}); + proverClient.finaliseBlock.mockResolvedValue({ block, proof }); publisher.processL2Block.mockResolvedValueOnce(true); globalVariableBuilder.buildGlobalVariables.mockResolvedValueOnce( new GlobalVariables(chainId, version, new Fr(lastBlockNumber + 1), Fr.ZERO, coinbase, feeRecipient), diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 2ef996d6c635..99c59da24894 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -306,7 +306,12 @@ export class Sequencer { const numRealTxs = txs.length; const pow2 = Math.log2(numRealTxs); const totalTxs = 2 ** Math.ceil(pow2); - const blockTicket = await this.prover.startNewBlock(Math.max(totalTxs, 2), globalVariables, l1ToL2Messages, emptyTx); + const blockTicket = await this.prover.startNewBlock( + Math.max(totalTxs, 2), + globalVariables, + l1ToL2Messages, + emptyTx, + ); for (const tx of txs) { await this.prover.addNewTx(tx); From 843226eb765807cae5e412f95a2fd82c5eb913d7 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 2 Apr 2024 15:38:53 +0100 Subject: [PATCH 04/24] Formatting --- yarn-project/end-to-end/src/integration_l1_publisher.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts index 994b95f324fa..510ccd414c8c 100644 --- a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts @@ -6,7 +6,7 @@ import { PROVING_STATUS, ProcessedTx, makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, - makeProcessedTx + makeProcessedTx, } from '@aztec/circuit-types'; import { EthAddress, From 9332d4f6b1b5972a2aceff6a4ea9cc8418048c07 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 2 Apr 2024 15:50:16 +0100 Subject: [PATCH 05/24] Formatting --- .../src/integration_l1_publisher.test.ts | 1 - yarn-project/prover-client/src/dummy-prover.ts | 2 +- .../src/orchestrator/block-building-helpers.ts | 2 +- .../src/orchestrator/orchestrator.test.ts | 18 ++++++++---------- .../src/orchestrator/orchestrator.ts | 4 ++-- .../src/orchestrator/proving-state.ts | 8 ++++---- .../prover-client/src/tx-prover/tx-prover.ts | 5 ++--- 7 files changed, 18 insertions(+), 22 deletions(-) diff --git a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts index 8755d78081ad..670dbfa7b573 100644 --- a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts @@ -13,7 +13,6 @@ import { // eslint-disable-next-line no-restricted-imports import { type ProcessedTx, - type ProvingSuccess, makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, makeProcessedTx, } from '@aztec/circuit-types'; diff --git a/yarn-project/prover-client/src/dummy-prover.ts b/yarn-project/prover-client/src/dummy-prover.ts index 43751918539a..c5a4a9b5ff9f 100644 --- a/yarn-project/prover-client/src/dummy-prover.ts +++ b/yarn-project/prover-client/src/dummy-prover.ts @@ -1,5 +1,5 @@ import { - BlockResult, + type BlockResult, L2Block, PROVING_STATUS, type ProcessedTx, diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index dcf73b8e39e6..f510ac65cd3d 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -5,7 +5,7 @@ import { BaseRollupInputs, ConstantRollupData, Fr, - L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, + type L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, MAX_NEW_NULLIFIERS_PER_TX, MAX_PUBLIC_DATA_READS_PER_TX, MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts index e672d5310c2a..a5daa606910a 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts @@ -1,10 +1,11 @@ import { - MerkleTreeId, - PROVING_STATUS, makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, makeProcessedTx, + MerkleTreeId, mockTx, - type ProcessedTx + PROVING_STATUS, + type ProcessedTx, + type ProvingFailure } from '@aztec/circuit-types'; import { AztecAddress, @@ -22,14 +23,14 @@ import { MAX_REVERTIBLE_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NULLIFIER_SUBTREE_HEIGHT, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, - PUBLIC_DATA_SUBTREE_HEIGHT, Proof, + PUBLIC_DATA_SUBTREE_HEIGHT, PublicDataTreeLeaf, PublicDataUpdateRequest, PublicKernelCircuitPublicInputs, SideEffect, - SideEffectLinkedToNoteHash, sideEffectCmp, + SideEffectLinkedToNoteHash, type BaseOrMergeRollupPublicInputs, type RootRollupPublicInputs, } from '@aztec/circuits.js'; @@ -52,14 +53,11 @@ import { MerkleTrees, type MerkleTreeOperations } from '@aztec/world-state'; import { mock, type MockProxy } from 'jest-mock-extended'; import { default as memdown, type MemDown } from 'memdown'; -import { createDebugLogger } from '@aztec/foundation/log'; import { getVerificationKeys } from '../mocks/verification_keys.js'; import { type RollupProver } from '../prover/index.js'; import { type RollupSimulator } from '../simulator/rollup.js'; import { ProvingOrchestrator } from './orchestrator.js'; -const logger = createDebugLogger('aztec:orchestrator-test'); - export const createMemDown = () => (memdown as any)() as MemDown; describe('prover/tx-prover', () => { @@ -505,7 +503,7 @@ describe('prover/tx-prover', () => { expect((result1 as ProvingFailure).reason).toBe('Proving cancelled'); } - builderDb.rollback(); + await builderDb.rollback(); const blockTicket2 = await builder.startNewBlock( 2, @@ -549,7 +547,7 @@ describe('prover/tx-prover', () => { await builder.addNewTx(txs1[0]); - builderDb.rollback(); + await builderDb.rollback(); const blockTicket2 = await builder.startNewBlock( 2, diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index f64dffb4402b..c990e93e14a6 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -103,9 +103,9 @@ export class ProvingOrchestrator { this.simulator = new RealRollupCircuitSimulator(simulationProvider); } - public static new(db: MerkleTreeOperations, simulationProvider: SimulationProvider, prover: RollupProver) { + public static async new(db: MerkleTreeOperations, simulationProvider: SimulationProvider, prover: RollupProver) { const orchestrator = new ProvingOrchestrator(db, simulationProvider, getVerificationKeys(), prover); - orchestrator.start(); + await orchestrator.start(); return Promise.resolve(orchestrator); } diff --git a/yarn-project/prover-client/src/orchestrator/proving-state.ts b/yarn-project/prover-client/src/orchestrator/proving-state.ts index 060e9fdc64c5..737fcd37dc05 100644 --- a/yarn-project/prover-client/src/orchestrator/proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/proving-state.ts @@ -1,8 +1,8 @@ -import { L2Block, type ProcessedTx, type ProvingResult } from '@aztec/circuit-types'; +import { type L2Block, type ProcessedTx, type ProvingResult } from '@aztec/circuit-types'; import { - AppendOnlyTreeSnapshot, - L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, - RootRollupPublicInputs, + type AppendOnlyTreeSnapshot, + type L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, + type RootRollupPublicInputs, type BaseOrMergeRollupPublicInputs, type Fr, type GlobalVariables, diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index af15e63119e2..dac887ddcfe3 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -1,5 +1,5 @@ import { type ProcessedTx } from '@aztec/circuit-types'; -import { BlockResult, type ProverClient, type ProvingTicket } from '@aztec/circuit-types/interfaces'; +import { type BlockResult, type ProverClient, type ProvingTicket } from '@aztec/circuit-types/interfaces'; import { type Fr, type GlobalVariables } from '@aztec/circuits.js'; import { type SimulationProvider } from '@aztec/simulator'; import { type WorldStateSynchronizer } from '@aztec/world-state'; @@ -32,8 +32,7 @@ export class TxProver implements ProverClient { * Starts the prover instance */ public start() { - this.orchestrator.start(); - return Promise.resolve(); + return this.orchestrator.start(); } /** From 6dec348c058c3e1110eb7eb5941445542ca8ab3e Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 2 Apr 2024 16:49:11 +0100 Subject: [PATCH 06/24] Merge fix --- yarn-project/end-to-end/src/integration_l1_publisher.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts index 670dbfa7b573..c2bba9532214 100644 --- a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts @@ -15,6 +15,7 @@ import { type ProcessedTx, makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, makeProcessedTx, + PROVING_STATUS, } from '@aztec/circuit-types'; import { EthAddress, From f6bc4cd81d76c0a6537d7c241f40f64e02906f05 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 2 Apr 2024 17:09:36 +0100 Subject: [PATCH 07/24] Formatting --- .../src/integration_l1_publisher.test.ts | 2 +- .../prover-client/src/dummy-prover.ts | 5 +- .../orchestrator/block-building-helpers.ts | 32 ++++-- .../src/orchestrator/orchestrator.test.ts | 107 +++++------------- .../src/orchestrator/orchestrator.ts | 35 ++---- .../src/orchestrator/proving-state.ts | 16 ++- .../prover-client/src/tx-prover/tx-prover.ts | 1 - 7 files changed, 72 insertions(+), 126 deletions(-) diff --git a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts index c2bba9532214..5ec84296ff86 100644 --- a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts @@ -12,10 +12,10 @@ import { } from '@aztec/aztec.js'; // eslint-disable-next-line no-restricted-imports import { + PROVING_STATUS, type ProcessedTx, makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, makeProcessedTx, - PROVING_STATUS, } from '@aztec/circuit-types'; import { EthAddress, diff --git a/yarn-project/prover-client/src/dummy-prover.ts b/yarn-project/prover-client/src/dummy-prover.ts index c5a4a9b5ff9f..1b343104ce4a 100644 --- a/yarn-project/prover-client/src/dummy-prover.ts +++ b/yarn-project/prover-client/src/dummy-prover.ts @@ -11,8 +11,6 @@ import { type GlobalVariables, makeEmptyProof } from '@aztec/circuits.js'; import { type Fr } from '@aztec/foundation/fields'; export class DummyProver implements ProverClient { - - public start(): Promise { return Promise.resolve(); } @@ -44,8 +42,7 @@ export class DummyProver implements ProverClient { return Promise.resolve(); } - cancelBlock(): void { - } + cancelBlock(): void {} finaliseBlock(): Promise { return Promise.resolve({ diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index f510ac65cd3d..1b39bc5d3c5c 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -2,9 +2,12 @@ import { MerkleTreeId, type ProcessedTx } from '@aztec/circuit-types'; import { ARCHIVE_HEIGHT, AppendOnlyTreeSnapshot, + type BaseOrMergeRollupPublicInputs, + type BaseParityInputs, BaseRollupInputs, ConstantRollupData, Fr, + type GlobalVariables, type L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, MAX_NEW_NULLIFIERS_PER_TX, MAX_PUBLIC_DATA_READS_PER_TX, @@ -16,12 +19,14 @@ import { NULLIFIER_SUBTREE_HEIGHT, NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, NULLIFIER_TREE_HEIGHT, + type NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NullifierLeafPreimage, PUBLIC_DATA_SUBTREE_HEIGHT, PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, PUBLIC_DATA_TREE_HEIGHT, PartialStateReference, PreviousRollupData, + type Proof, PublicDataTreeLeaf, PublicDataTreeLeafPreimage, ROLLUP_VK_TREE_HEIGHT, @@ -29,25 +34,20 @@ import { RollupKernelData, RollupTypes, RootParityInput, - RootRollupInputs, - StateDiffHints, - VK_TREE_HEIGHT, - type BaseOrMergeRollupPublicInputs, - type BaseParityInputs, - type GlobalVariables, - type NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, - type Proof, type RootParityInputs, + RootRollupInputs, type RootRollupPublicInputs, + StateDiffHints, type StateReference, - type VerificationKey + VK_TREE_HEIGHT, + type VerificationKey, } from '@aztec/circuits.js'; import { assertPermutation, makeTuple } from '@aztec/foundation/array'; import { type DebugLogger } from '@aztec/foundation/log'; -import { assertLength, toFriendlyJSON, type Tuple } from '@aztec/foundation/serialize'; +import { type Tuple, assertLength, toFriendlyJSON } from '@aztec/foundation/serialize'; import { type MerkleTreeOperations } from '@aztec/world-state'; -import { getVerificationKeys, type VerificationKeys } from '../mocks/verification_keys.js'; +import { type VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; import { type RollupProver } from '../prover/index.js'; import { type RollupSimulator } from '../simulator/rollup.js'; @@ -212,7 +212,15 @@ export async function executeRootRollupCircuit( logger?: DebugLogger, ): Promise<[RootRollupPublicInputs, Proof]> { logger?.debug(`Running root rollup circuit`); - const rootInput = await getRootRollupInput(...left, ...right, l1ToL2Roots, newL1ToL2Messages, messageTreeSnapshot, messageTreeRootSiblingPath, db); + const rootInput = await getRootRollupInput( + ...left, + ...right, + l1ToL2Roots, + newL1ToL2Messages, + messageTreeSnapshot, + messageTreeRootSiblingPath, + db, + ); // Simulate and get proof for the root circuit const rootOutput = await simulator.rootRollupCircuit(rootInput); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts index a5daa606910a..0cf37192a6c4 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts @@ -1,14 +1,15 @@ import { - makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, - makeProcessedTx, MerkleTreeId, - mockTx, PROVING_STATUS, type ProcessedTx, - type ProvingFailure + type ProvingFailure, + makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots, + makeProcessedTx, + mockTx, } from '@aztec/circuit-types'; import { AztecAddress, + type BaseOrMergeRollupPublicInputs, EthAddress, Fr, GlobalVariables, @@ -23,16 +24,15 @@ import { MAX_REVERTIBLE_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NULLIFIER_SUBTREE_HEIGHT, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, - Proof, PUBLIC_DATA_SUBTREE_HEIGHT, + Proof, PublicDataTreeLeaf, PublicDataUpdateRequest, PublicKernelCircuitPublicInputs, + type RootRollupPublicInputs, SideEffect, - sideEffectCmp, SideEffectLinkedToNoteHash, - type BaseOrMergeRollupPublicInputs, - type RootRollupPublicInputs, + sideEffectCmp, } from '@aztec/circuits.js'; import { fr, @@ -48,10 +48,10 @@ import { padArrayEnd, times } from '@aztec/foundation/collection'; import { sleep } from '@aztec/foundation/sleep'; import { openTmpStore } from '@aztec/kv-store/utils'; import { WASMSimulator } from '@aztec/simulator'; -import { MerkleTrees, type MerkleTreeOperations } from '@aztec/world-state'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; -import { mock, type MockProxy } from 'jest-mock-extended'; -import { default as memdown, type MemDown } from 'memdown'; +import { type MockProxy, mock } from 'jest-mock-extended'; +import { type MemDown, default as memdown } from 'memdown'; import { getVerificationKeys } from '../mocks/verification_keys.js'; import { type RollupProver } from '../prover/index.js'; @@ -85,7 +85,7 @@ describe('prover/tx-prover', () => { const makeGlobals = (blockNumber: number) => { return new GlobalVariables(chainId, version, new Fr(blockNumber), Fr.ZERO, coinbase, feeRecipient); - } + }; beforeEach(async () => { blockNumber = 3; @@ -392,12 +392,12 @@ describe('prover/tx-prover', () => { // This will need to be a 2 tx block const blockTicket = await builder.startNewBlock(2, globals, [], emptyTx); - + await builder.addNewTx(tx); - + // we need to complete the block as we have not added a full set of txs await builder.setBlockCompleted(); - + const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); @@ -467,27 +467,16 @@ describe('prover/tx-prover', () => { }, 200_000); it('cancels current block and switches to new ones', async () => { - const txs1 = await Promise.all([ - makeBloatedProcessedTx(1), - makeBloatedProcessedTx(2), - ]); + const txs1 = await Promise.all([makeBloatedProcessedTx(1), makeBloatedProcessedTx(2)]); - const txs2 = await Promise.all([ - makeBloatedProcessedTx(3), - makeBloatedProcessedTx(4), - ]); + const txs2 = await Promise.all([makeBloatedProcessedTx(3), makeBloatedProcessedTx(4)]); const globals1: GlobalVariables = makeGlobals(100); const globals2: GlobalVariables = makeGlobals(101); const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); - const blockTicket1 = await builder.startNewBlock( - 2, - globals1, - l1ToL2Messages, - await makeEmptyProcessedTx(), - ); + const blockTicket1 = await builder.startNewBlock(2, globals1, l1ToL2Messages, await makeEmptyProcessedTx()); await builder.addNewTx(txs1[0]); await builder.addNewTx(txs1[1]); @@ -501,16 +490,11 @@ describe('prover/tx-prover', () => { // however it may have actually completed proving before we cancelled in which case it could be a succes code if (result1.status === PROVING_STATUS.FAILURE) { expect((result1 as ProvingFailure).reason).toBe('Proving cancelled'); - } + } await builderDb.rollback(); - const blockTicket2 = await builder.startNewBlock( - 2, - globals2, - l1ToL2Messages, - await makeEmptyProcessedTx(), - ); + const blockTicket2 = await builder.startNewBlock(2, globals2, l1ToL2Messages, await makeEmptyProcessedTx()); await builder.addNewTx(txs2[0]); await builder.addNewTx(txs2[1]); @@ -523,38 +507,22 @@ describe('prover/tx-prover', () => { }, 10000); it('automatically cancels an incomplete block when starting a new one', async () => { - const txs1 = await Promise.all([ - makeBloatedProcessedTx(1), - makeBloatedProcessedTx(2), - ]); + const txs1 = await Promise.all([makeBloatedProcessedTx(1), makeBloatedProcessedTx(2)]); - const txs2 = await Promise.all([ - makeBloatedProcessedTx(3), - makeBloatedProcessedTx(4), - ]); + const txs2 = await Promise.all([makeBloatedProcessedTx(3), makeBloatedProcessedTx(4)]); const globals1: GlobalVariables = makeGlobals(100); const globals2: GlobalVariables = makeGlobals(101); const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); - const blockTicket1 = await builder.startNewBlock( - 2, - globals1, - l1ToL2Messages, - await makeEmptyProcessedTx(), - ); + const blockTicket1 = await builder.startNewBlock(2, globals1, l1ToL2Messages, await makeEmptyProcessedTx()); await builder.addNewTx(txs1[0]); await builderDb.rollback(); - const blockTicket2 = await builder.startNewBlock( - 2, - globals2, - l1ToL2Messages, - await makeEmptyProcessedTx(), - ); + const blockTicket2 = await builder.startNewBlock(2, globals2, l1ToL2Messages, await makeEmptyProcessedTx()); await builder.addNewTx(txs2[0]); await builder.addNewTx(txs2[1]); @@ -576,12 +544,7 @@ describe('prover/tx-prover', () => { const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); // this needs to be a 4 tx block that will need to be completed - const blockTicket = await builder.startNewBlock( - 4, - globalVariables, - l1ToL2Messages, - await makeEmptyProcessedTx(), - ); + const blockTicket = await builder.startNewBlock(4, globalVariables, l1ToL2Messages, await makeEmptyProcessedTx()); for (const tx of txs) { await builder.addNewTx(tx); @@ -640,11 +603,7 @@ describe('prover/tx-prover', () => { }, 1000); it('throws if finalising an already finalised block', async () => { - - const txs = await Promise.all([ - makeEmptyProcessedTx(), - makeEmptyProcessedTx(), - ]); + const txs = await Promise.all([makeEmptyProcessedTx(), makeEmptyProcessedTx()]); const blockTicket = await builder.startNewBlock(txs.length, globalVariables, [], await makeEmptyProcessedTx()); @@ -656,9 +615,7 @@ describe('prover/tx-prover', () => { expect(result.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); expect(finalisedBlock.l2Block.number).toEqual(blockNumber); - await expect(async () => await builder.finaliseBlock()).rejects.toThrow( - 'Block already finalised', - ); + await expect(async () => await builder.finaliseBlock()).rejects.toThrow('Block already finalised'); }, 20000); it('throws if adding to a cancelled block', async () => { @@ -669,17 +626,9 @@ describe('prover/tx-prover', () => { await expect(async () => await builder.addNewTx(await makeEmptyProcessedTx())).rejects.toThrow( 'Rollup not accepting further transactions', ); - }, 10000); - it.each([ - [-4], - [0], - [1], - [3], - [8.1], - [7] - ] as const)( + it.each([[-4], [0], [1], [3], [8.1], [7]] as const)( 'fails to start a block with %i transaxctions', async (blockSize: number) => { await expect( diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index c990e93e14a6..0f96c93d492e 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -8,14 +8,15 @@ import { type BaseRollupInputs, Fr, type GlobalVariables, + L1_TO_L2_MSG_SUBTREE_HEIGHT, + L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NUM_BASE_PARITY_PER_ROOT_PARITY, type Proof, type RootParityInput, RootParityInputs, - L1_TO_L2_MSG_SUBTREE_HEIGHT, - L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, } from '@aztec/circuits.js'; +import { makeTuple } from '@aztec/foundation/array'; import { padArrayEnd } from '@aztec/foundation/collection'; import { MemoryFifo } from '@aztec/foundation/fifo'; import { createDebugLogger } from '@aztec/foundation/log'; @@ -44,7 +45,6 @@ import { validateTx, } from './block-building-helpers.js'; import { type MergeRollupInputData, ProvingState } from './proving-state.js'; -import { makeTuple } from '@aztec/foundation/array'; const logger = createDebugLogger('aztec:prover:proving-orchestrator'); @@ -160,10 +160,11 @@ export class ProvingOrchestrator { L1_TO_L2_MSG_SUBTREE_HEIGHT, this.db, ); - + const newL1ToL2MessageTreeRootSiblingPath = makeTuple( L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, - i => (i < newL1ToL2MessageTreeRootSiblingPathArray.length ? newL1ToL2MessageTreeRootSiblingPathArray[i] : Fr.ZERO), + i => + i < newL1ToL2MessageTreeRootSiblingPathArray.length ? newL1ToL2MessageTreeRootSiblingPathArray[i] : Fr.ZERO, 0, ); @@ -232,11 +233,7 @@ export class ProvingOrchestrator { // we need to pad the rollup with empty transactions for (let i = this.provingState.transactionsReceived; i < this.provingState.totalNumTxs; i++) { const paddingTxIndex = this.provingState.addNewTx(this.provingState.emptyTx); - await this.prepareBaseRollupInputs( - this.provingState, - BigInt(paddingTxIndex), - this.provingState!.emptyTx, - ); + await this.prepareBaseRollupInputs(this.provingState, BigInt(paddingTxIndex), this.provingState!.emptyTx); } } @@ -286,7 +283,7 @@ export class ProvingOrchestrator { return { l2Block, proof: this.provingState.finalProof, - } + }; } /** @@ -318,11 +315,7 @@ export class ProvingOrchestrator { } // Updates the merkle trees for a transaction. The first enqueued job for a transaction - private async prepareBaseRollupInputs( - provingState: ProvingState | undefined, - index: bigint, - tx: ProcessedTx, - ) { + private async prepareBaseRollupInputs(provingState: ProvingState | undefined, index: bigint, tx: ProcessedTx) { if (!provingState?.verifyState()) { logger('Not preparing base rollup inputs, state invalid'); return; @@ -429,9 +422,7 @@ export class ProvingOrchestrator { } // Executes the root rollup circuit - private async runRootRollup( - provingState: ProvingState | undefined, - ) { + private async runRootRollup(provingState: ProvingState | undefined) { if (!provingState?.verifyState()) { logger('Not running root rollup, state no longer valid'); return; @@ -528,11 +519,7 @@ export class ProvingOrchestrator { logger('Not ready for root rollup'); return; } - this.enqueueJob(provingState, PROVING_JOB_TYPE.ROOT_ROLLUP, () => - this.runRootRollup( - provingState, - ), - ); + this.enqueueJob(provingState, PROVING_JOB_TYPE.ROOT_ROLLUP, () => this.runRootRollup(provingState)); } private storeAndExecuteNextMergeLevel( diff --git a/yarn-project/prover-client/src/orchestrator/proving-state.ts b/yarn-project/prover-client/src/orchestrator/proving-state.ts index 737fcd37dc05..4201de80a358 100644 --- a/yarn-project/prover-client/src/orchestrator/proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/proving-state.ts @@ -1,18 +1,17 @@ import { type L2Block, type ProcessedTx, type ProvingResult } from '@aztec/circuit-types'; import { type AppendOnlyTreeSnapshot, - type L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, - type RootRollupPublicInputs, type BaseOrMergeRollupPublicInputs, type Fr, type GlobalVariables, + type L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, type NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, type Proof, type RootParityInput, + type RootRollupPublicInputs, } from '@aztec/circuits.js'; import { type Tuple } from '@aztec/foundation/serialize'; - export type MergeRollupInputData = { inputs: [BaseOrMergeRollupPublicInputs | undefined, BaseOrMergeRollupPublicInputs | undefined]; proofs: [Proof | undefined, Proof | undefined]; @@ -86,7 +85,10 @@ export class ProvingState { } public verifyState() { - return this.provingStateLifecyle === PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED || this.provingStateLifecyle === PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL; + return ( + this.provingStateLifecyle === PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED || + this.provingStateLifecyle === PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL + ); } public isAcceptingTransactions() { @@ -123,7 +125,11 @@ export class ProvingState { } public isReadyForRootRollup() { - return !(this.mergeRollupInputs[0] === undefined || this.finalRootParityInput === undefined || this.mergeRollupInputs[0].inputs.findIndex(p => !p) !== -1); + return !( + this.mergeRollupInputs[0] === undefined || + this.finalRootParityInput === undefined || + this.mergeRollupInputs[0].inputs.findIndex(p => !p) !== -1 + ); } public setRootParityInputs(inputs: RootParityInput, index: number) { diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index dac887ddcfe3..76fe4d7f376c 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -27,7 +27,6 @@ export class TxProver implements ProverClient { ); } - /** * Starts the prover instance */ From e44e2b324fc9eed77d65a55ceb96d337d658f4d6 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 2 Apr 2024 21:20:43 +0100 Subject: [PATCH 08/24] Fixes --- yarn-project/circuit-types/src/interfaces/block-prover.ts | 2 ++ yarn-project/prover-client/src/dummy-prover.ts | 4 ++++ yarn-project/prover-client/src/tx-prover/tx-prover.ts | 4 ++++ yarn-project/sequencer-client/src/sequencer/sequencer.ts | 2 ++ 4 files changed, 12 insertions(+) diff --git a/yarn-project/circuit-types/src/interfaces/block-prover.ts b/yarn-project/circuit-types/src/interfaces/block-prover.ts index d163afaee047..4ba58e123e64 100644 --- a/yarn-project/circuit-types/src/interfaces/block-prover.ts +++ b/yarn-project/circuit-types/src/interfaces/block-prover.ts @@ -45,4 +45,6 @@ export interface BlockProver { cancelBlock(): void; finaliseBlock(): Promise; + + setBlockCompleted(): Promise; } diff --git a/yarn-project/prover-client/src/dummy-prover.ts b/yarn-project/prover-client/src/dummy-prover.ts index 1b343104ce4a..de61267c7384 100644 --- a/yarn-project/prover-client/src/dummy-prover.ts +++ b/yarn-project/prover-client/src/dummy-prover.ts @@ -50,4 +50,8 @@ export class DummyProver implements ProverClient { proof: makeEmptyProof(), }); } + + setBlockCompleted(): Promise { + return Promise.resolve(); + } } diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index 76fe4d7f376c..91a196adcd37 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -78,4 +78,8 @@ export class TxProver implements ProverClient { public finaliseBlock(): Promise { return this.finaliseBlock(); } + + public setBlockCompleted(): Promise { + return this.orchestrator.setBlockCompleted(); + } } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 95bb307cf3a8..92754ae48e0b 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -317,6 +317,8 @@ export class Sequencer { await this.prover.addNewTx(tx); } + await this.prover.setBlockCompleted(); + const result = await blockTicket.provingPromise; if (result.status === PROVING_STATUS.FAILURE) { throw new Error(`Block proving failed, reason: ${result.reason}`); From c1987967b6730b1498e2cc3adffc9497f6ac3dc6 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 2 Apr 2024 21:37:35 +0100 Subject: [PATCH 09/24] Fix --- .../prover-client/src/orchestrator/orchestrator.ts | 10 ++++++---- yarn-project/prover-client/src/tx-prover/tx-prover.ts | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index 0f96c93d492e..08193162bb25 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -1,5 +1,5 @@ import { Body, L2Block, MerkleTreeId, type ProcessedTx, type TxEffect, toTxEffect } from '@aztec/circuit-types'; -import { PROVING_STATUS, type ProvingResult, type ProvingTicket } from '@aztec/circuit-types/interfaces'; +import { BlockResult, PROVING_STATUS, type ProvingResult, type ProvingTicket } from '@aztec/circuit-types/interfaces'; import { type CircuitSimulationStats } from '@aztec/circuit-types/stats'; import { type AppendOnlyTreeSnapshot, @@ -280,10 +280,12 @@ export class ProvingOrchestrator { this.provingState.block = l2Block; - return { - l2Block, + const blockResult: BlockResult = { proof: this.provingState.finalProof, - }; + block: l2Block, + } + + return blockResult; } /** diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index 91a196adcd37..1b4e25419ba3 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -76,7 +76,7 @@ export class TxProver implements ProverClient { this.orchestrator.cancelBlock(); } public finaliseBlock(): Promise { - return this.finaliseBlock(); + return this.orchestrator.finaliseBlock(); } public setBlockCompleted(): Promise { From a60162a962461c1424684477ef5be363a287997e Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 2 Apr 2024 21:51:10 +0100 Subject: [PATCH 10/24] Fixes --- .../src/orchestrator/orchestrator.test.ts | 24 +++++++++---------- .../src/orchestrator/orchestrator.ts | 9 +++++-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts index 0cf37192a6c4..6cc10bf1f6e7 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts @@ -320,7 +320,7 @@ describe('prover/tx-prover', () => { const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(blockNumber); + expect(finalisedBlock.block.number).toEqual(blockNumber); await updateExpectedTreesFromTxs(txs); const noteHashTreeAfter = await builderDb.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); @@ -353,7 +353,7 @@ describe('prover/tx-prover', () => { expect(result.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(blockNumber); + expect(finalisedBlock.block.number).toEqual(blockNumber); }, 30_000); it('builds a block with 1 transaction', async () => { @@ -373,7 +373,7 @@ describe('prover/tx-prover', () => { expect(result.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(blockNumber); + expect(finalisedBlock.block.number).toEqual(blockNumber); }, 30_000); it('builds multiple blocks in sequence', async () => { @@ -402,8 +402,8 @@ describe('prover/tx-prover', () => { expect(result.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(blockNum); - header = finalisedBlock.l2Block.header; + expect(finalisedBlock.block.number).toEqual(blockNum); + header = finalisedBlock.block.header; await builderDb.commit(); } @@ -434,7 +434,7 @@ describe('prover/tx-prover', () => { expect(result.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(blockNumber); + expect(finalisedBlock.block.number).toEqual(blockNumber); }, 200_000); it('builds a block concurrently with transactions', async () => { @@ -463,7 +463,7 @@ describe('prover/tx-prover', () => { expect(result.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(blockNumber); + expect(finalisedBlock.block.number).toEqual(blockNumber); }, 200_000); it('cancels current block and switches to new ones', async () => { @@ -503,7 +503,7 @@ describe('prover/tx-prover', () => { expect(result2.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(101); + expect(finalisedBlock.block.number).toEqual(101); }, 10000); it('automatically cancels an incomplete block when starting a new one', async () => { @@ -535,7 +535,7 @@ describe('prover/tx-prover', () => { expect(result2.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(101); + expect(finalisedBlock.block.number).toEqual(101); }, 10000); it('builds an unbalanced L2 block', async () => { @@ -556,7 +556,7 @@ describe('prover/tx-prover', () => { expect(result.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(blockNumber); + expect(finalisedBlock.block.number).toEqual(blockNumber); }, 200_000); it('throws if adding too many transactions', async () => { @@ -581,7 +581,7 @@ describe('prover/tx-prover', () => { expect(result.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(blockNumber); + expect(finalisedBlock.block.number).toEqual(blockNumber); }, 30_000); it('throws if adding a transaction before start', async () => { @@ -614,7 +614,7 @@ describe('prover/tx-prover', () => { const result = await blockTicket.provingPromise; expect(result.status).toBe(PROVING_STATUS.SUCCESS); const finalisedBlock = await builder.finaliseBlock(); - expect(finalisedBlock.l2Block.number).toEqual(blockNumber); + expect(finalisedBlock.block.number).toEqual(blockNumber); await expect(async () => await builder.finaliseBlock()).rejects.toThrow('Block already finalised'); }, 20000); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index 08193162bb25..c092c5c1a9c4 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -1,5 +1,10 @@ import { Body, L2Block, MerkleTreeId, type ProcessedTx, type TxEffect, toTxEffect } from '@aztec/circuit-types'; -import { BlockResult, PROVING_STATUS, type ProvingResult, type ProvingTicket } from '@aztec/circuit-types/interfaces'; +import { + type BlockResult, + PROVING_STATUS, + type ProvingResult, + type ProvingTicket, +} from '@aztec/circuit-types/interfaces'; import { type CircuitSimulationStats } from '@aztec/circuit-types/stats'; import { type AppendOnlyTreeSnapshot, @@ -283,7 +288,7 @@ export class ProvingOrchestrator { const blockResult: BlockResult = { proof: this.provingState.finalProof, block: l2Block, - } + }; return blockResult; } From cfbdcd6ddb1ab269645a85e1254f9d361e14328b Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 3 Apr 2024 11:55:00 +0100 Subject: [PATCH 11/24] Test fix --- yarn-project/end-to-end/src/integration_l1_publisher.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts index 5ec84296ff86..7beed5a51e3a 100644 --- a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts @@ -143,6 +143,7 @@ describe('L1Publisher integration', () => { l2QueueSize: 10, }; const worldStateSynchronizer = new ServerWorldStateSynchronizer(tmpStore, builderDb, blockSource, worldStateConfig); + await worldStateSynchronizer.start(); builder = await TxProver.new({}, worldStateSynchronizer, new WASMSimulator()); l2Proof = Buffer.alloc(0); @@ -394,6 +395,8 @@ describe('L1Publisher integration', () => { const blockResult = await builder.finaliseBlock(); const block = blockResult.block; prevHeader = block.header; + blockSource.getL1ToL2Messages.mockResolvedValueOnce(currentL1ToL2Messages); + blockSource.getBlocks.mockResolvedValueOnce([block]); const newL2ToL1MsgsArray = block.body.txEffects.flatMap(txEffect => txEffect.l2ToL1Msgs); @@ -486,6 +489,8 @@ describe('L1Publisher integration', () => { const blockResult = await builder.finaliseBlock(); const block = blockResult.block; prevHeader = block.header; + blockSource.getL1ToL2Messages.mockResolvedValueOnce(l1ToL2Messages); + blockSource.getBlocks.mockResolvedValueOnce([block]); writeJson(`empty_block_${i}`, block, [], AztecAddress.ZERO, deployerAccount.address); From f87f18237b4928a15355ced35929a43eefab8f1e Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 5 Apr 2024 12:47:18 +0100 Subject: [PATCH 12/24] Simulate and prove concurrently --- .../src/sequencer/public_processor.ts | 18 +++++- .../src/sequencer/sequencer.ts | 62 +++++++++++++------ 2 files changed, 59 insertions(+), 21 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/public_processor.ts b/yarn-project/sequencer-client/src/sequencer/public_processor.ts index 14c15d70b9bd..f74a843d10b3 100644 --- a/yarn-project/sequencer-client/src/sequencer/public_processor.ts +++ b/yarn-project/sequencer-client/src/sequencer/public_processor.ts @@ -8,6 +8,7 @@ import { makeProcessedTx, toTxEffect, validateProcessedTx, + BlockProver, } from '@aztec/circuit-types'; import { type TxSequencerProcessingStats } from '@aztec/circuit-types/stats'; import { type GlobalVariables, type Header } from '@aztec/circuits.js'; @@ -23,6 +24,7 @@ import { ContractsDataSourcePublicDB, WorldStateDB, WorldStatePublicDB } from '. import { RealPublicKernelCircuitSimulator } from '../simulator/public_kernel.js'; import { type AbstractPhaseManager, PublicKernelPhase } from './abstract_phase_manager.js'; import { PhaseManagerFactory } from './phase_manager_factory.js'; +import { TxValidator } from './tx_validator.js'; /** * Creates new instances of PublicProcessor given the provided merkle tree db and contract data source. @@ -85,7 +87,7 @@ export class PublicProcessor { * @param txs - Txs to process. * @returns The list of processed txs with their circuit simulation outputs. */ - public async process(txs: Tx[]): Promise<[ProcessedTx[], FailedTx[], ProcessReturnValues[]]> { + public async process(txs: Tx[], maxTransactions = txs.length, blockProver?: BlockProver, txValidator?: TxValidator): Promise<[ProcessedTx[], FailedTx[], ProcessReturnValues[]]> { // The processor modifies the tx objects in place, so we need to clone them. txs = txs.map(tx => Tx.clone(tx)); const result: ProcessedTx[] = []; @@ -93,6 +95,9 @@ export class PublicProcessor { const returns: ProcessReturnValues[] = []; for (const tx of txs) { + if (result.length >= maxTransactions) { + break; + } let returnValues: ProcessReturnValues = undefined; let phase: AbstractPhaseManager | undefined = PhaseManagerFactory.phaseFromTx( tx, @@ -133,7 +138,6 @@ export class PublicProcessor { const processedTransaction = makeProcessedTx(tx, publicKernelPublicInput, proof, revertReason); validateProcessedTx(processedTransaction); - result.push(processedTransaction); returns.push(returnValues); this.log(`Processed public part of ${tx.data.endNonRevertibleData.newNullifiers[0].value}`, { @@ -145,6 +149,16 @@ export class PublicProcessor { 0, ...tx.getStats(), } satisfies TxSequencerProcessingStats); + if (txValidator) { + const [_, invalid] = await txValidator.validateTxs([processedTransaction]); + if (invalid.length) { + throw new Error(`Transaction ${invalid[0].hash} invalid after public processing`); + } + } + if (blockProver) { + await blockProver?.addNewTx(processedTransaction); + } + result.push(processedTransaction); } catch (err: any) { const errorMessage = err instanceof Error ? err.message : 'Unknown error'; this.log.warn(`Failed to process tx ${tx.getTxHash()}: ${errorMessage}`); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 92754ae48e0b..d8d352221b76 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -193,9 +193,28 @@ export class Sequencer { this.log.info(`Building block ${newBlockNumber} with ${validTxs.length} transactions`); this.state = SequencerState.CREATING_BLOCK; + // Get l1 to l2 messages from the contract + this.log('Requesting L1 to L2 messages from contract'); + const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(newBlockNumber)); + this.log(`Retrieved ${l1ToL2Messages.length} L1 to L2 messages for block ${newBlockNumber}`); + // We create a fresh processor each time to reset any cached state (eg storage writes) const processor = await this.publicProcessorFactory.create(historicalHeader, newGlobalVariables); - const [publicProcessorDuration, [processedTxs, failedTxs]] = await elapsed(() => processor.process(validTxs)); + + const emptyTx = processor.makeEmptyProcessedTx(); + + const numRealTxs = validTxs.length; + const pow2 = Math.log2(numRealTxs); + const totalTxs = 2 ** Math.ceil(pow2); + const blockSize = Math.max(2, totalTxs); + const blockTicket = await this.prover.startNewBlock( + blockSize, + newGlobalVariables, + l1ToL2Messages, + emptyTx, + ); + + const [publicProcessorDuration, [processedTxs, failedTxs]] = await elapsed(() => processor.process(validTxs, blockSize, this.prover, txValidator)); if (failedTxs.length > 0) { const failedTxData = failedTxs.map(fail => fail.tx); this.log(`Dropping failed txs ${Tx.getHashes(failedTxData).join(', ')}`); @@ -206,37 +225,42 @@ export class Sequencer { // public functions emitting nullifiers would pass earlier check but fail here. // Note that we're checking all nullifiers generated in the private execution twice, // we could store the ones already checked and skip them here as an optimization. - const processedValidTxs = await this.takeValidTxs(processedTxs, txValidator); + const processedValidTxs = processedTxs; if (processedValidTxs.length === 0) { this.log('No txs processed correctly to build block. Exiting'); + this.prover.cancelBlock(); return; } - await assertBlockHeight(); + await this.prover.setBlockCompleted(); - // Get l1 to l2 messages from the contract - this.log('Requesting L1 to L2 messages from contract'); - const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(newBlockNumber)); - this.log(`Retrieved ${l1ToL2Messages.length} L1 to L2 messages for block ${newBlockNumber}`); + const result = await blockTicket.provingPromise; + if (result.status === PROVING_STATUS.FAILURE) { + throw new Error(`Block proving failed, reason: ${result.reason}`); + } + const blockResult = await this.prover.finaliseBlock(); + const block = blockResult.block; + + await assertBlockHeight(); // Build the new block by running the rollup circuits this.log(`Assembling block with txs ${processedValidTxs.map(tx => tx.hash).join(', ')}`); await assertBlockHeight(); - const emptyTx = processor.makeEmptyProcessedTx(); - const [rollupCircuitsDuration, block] = await elapsed(() => - this.buildBlock(processedValidTxs, l1ToL2Messages, emptyTx, newGlobalVariables), - ); - - this.log(`Assembled block ${block.number}`, { - eventName: 'l2-block-built', - duration: workTimer.ms(), - publicProcessDuration: publicProcessorDuration, - rollupCircuitsDuration: rollupCircuitsDuration, - ...block.getStats(), - } satisfies L2BlockBuiltStats); + + // const [rollupCircuitsDuration, block] = await elapsed(() => + // this.buildBlock(processedValidTxs, l1ToL2Messages, emptyTx, newGlobalVariables), + // ); + + // this.log(`Assembled block ${block.number}`, { + // eventName: 'l2-block-built', + // duration: workTimer.ms(), + // publicProcessDuration: publicProcessorDuration, + // rollupCircuitsDuration: rollupCircuitsDuration, + // ...block.getStats(), + // } satisfies L2BlockBuiltStats); await assertBlockHeight(); From e5eda9e31d601faa497b45499629749800b45757 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 7 Apr 2024 13:04:47 +0100 Subject: [PATCH 13/24] Fixes --- .../orchestrator/block-building-helpers.ts | 26 ++++---- .../src/sequencer/public_processor.ts | 11 +++- .../src/sequencer/sequencer.ts | 66 ++++--------------- 3 files changed, 34 insertions(+), 69 deletions(-) diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index 324ba7a8d256..6cbc988145a5 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -2,11 +2,14 @@ import { MerkleTreeId, type ProcessedTx } from '@aztec/circuit-types'; import { ARCHIVE_HEIGHT, AppendOnlyTreeSnapshot, + type BaseOrMergeRollupPublicInputs, + type BaseParityInputs, BaseRollupInputs, ConstantRollupData, Fr, + type GlobalVariables, KernelData, - L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, + type L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, MAX_NEW_NULLIFIERS_PER_TX, MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, MembershipWitness, @@ -16,36 +19,33 @@ import { NULLIFIER_SUBTREE_HEIGHT, NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, NULLIFIER_TREE_HEIGHT, + type NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NullifierLeafPreimage, PUBLIC_DATA_SUBTREE_HEIGHT, PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, PUBLIC_DATA_TREE_HEIGHT, PartialStateReference, PreviousRollupData, + type Proof, PublicDataTreeLeaf, + type PublicDataTreeLeafPreimage, ROLLUP_VK_TREE_HEIGHT, RollupTypes, RootParityInput, - RootRollupInputs, - StateDiffHints, - VK_TREE_HEIGHT, - type BaseOrMergeRollupPublicInputs, - type BaseParityInputs, - type GlobalVariables, - type NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, - type Proof, - type PublicDataTreeLeafPreimage, type RootParityInputs, + RootRollupInputs, type RootRollupPublicInputs, + StateDiffHints, type StateReference, - type VerificationKey + VK_TREE_HEIGHT, + type VerificationKey, } from '@aztec/circuits.js'; import { assertPermutation, makeTuple } from '@aztec/foundation/array'; import { type DebugLogger } from '@aztec/foundation/log'; -import { assertLength, toFriendlyJSON, type Tuple } from '@aztec/foundation/serialize'; +import { type Tuple, assertLength, toFriendlyJSON } from '@aztec/foundation/serialize'; import { type MerkleTreeOperations } from '@aztec/world-state'; -import { getVerificationKeys, type VerificationKeys } from '../mocks/verification_keys.js'; +import { type VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; import { type RollupProver } from '../prover/index.js'; import { type RollupSimulator } from '../simulator/rollup.js'; diff --git a/yarn-project/sequencer-client/src/sequencer/public_processor.ts b/yarn-project/sequencer-client/src/sequencer/public_processor.ts index 7a03bfd12df5..9da7db2bc4e8 100644 --- a/yarn-project/sequencer-client/src/sequencer/public_processor.ts +++ b/yarn-project/sequencer-client/src/sequencer/public_processor.ts @@ -1,4 +1,5 @@ import { + type BlockProver, type FailedTx, type ProcessedTx, type SimulationError, @@ -7,7 +8,6 @@ import { makeProcessedTx, toTxEffect, validateProcessedTx, - BlockProver, } from '@aztec/circuit-types'; import { type TxSequencerProcessingStats } from '@aztec/circuit-types/stats'; import { type GlobalVariables, type Header, type KernelCircuitPublicInputs } from '@aztec/circuits.js'; @@ -23,7 +23,7 @@ import { ContractsDataSourcePublicDB, WorldStateDB, WorldStatePublicDB } from '. import { RealPublicKernelCircuitSimulator } from '../simulator/public_kernel.js'; import { type AbstractPhaseManager, PublicKernelPhase } from './abstract_phase_manager.js'; import { PhaseManagerFactory } from './phase_manager_factory.js'; -import { TxValidator } from './tx_validator.js'; +import { type TxValidator } from './tx_validator.js'; /** * Creates new instances of PublicProcessor given the provided merkle tree db and contract data source. @@ -86,7 +86,12 @@ export class PublicProcessor { * @param txs - Txs to process. * @returns The list of processed txs with their circuit simulation outputs. */ - public async process(txs: Tx[], maxTransactions = txs.length, blockProver?: BlockProver, txValidator?: TxValidator): Promise<[ProcessedTx[], FailedTx[], ProcessReturnValues[]]> { + public async process( + txs: Tx[], + maxTransactions = txs.length, + blockProver?: BlockProver, + txValidator?: TxValidator, + ): Promise<[ProcessedTx[], FailedTx[], ProcessReturnValues[]]> { // The processor modifies the tx objects in place, so we need to clone them. txs = txs.map(tx => Tx.clone(tx)); const result: ProcessedTx[] = []; diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index d8d352221b76..01c1e53747ef 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -1,7 +1,7 @@ -import { type L1ToL2MessageSource, type L2Block, type L2BlockSource, type ProcessedTx, Tx } from '@aztec/circuit-types'; -import { type BlockProver, PROVING_STATUS } from '@aztec/circuit-types/interfaces'; +import { Tx, type L1ToL2MessageSource, type L2Block, type L2BlockSource, type ProcessedTx } from '@aztec/circuit-types'; +import { PROVING_STATUS, type BlockProver } from '@aztec/circuit-types/interfaces'; import { type L2BlockBuiltStats } from '@aztec/circuit-types/stats'; -import { AztecAddress, EthAddress, type GlobalVariables } from '@aztec/circuits.js'; +import { AztecAddress, EthAddress } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; @@ -203,6 +203,8 @@ export class Sequencer { const emptyTx = processor.makeEmptyProcessedTx(); + const blockBuildingTimer = new Timer(); + const numRealTxs = validTxs.length; const pow2 = Math.log2(numRealTxs); const totalTxs = 2 ** Math.ceil(pow2); @@ -249,18 +251,13 @@ export class Sequencer { await assertBlockHeight(); - - // const [rollupCircuitsDuration, block] = await elapsed(() => - // this.buildBlock(processedValidTxs, l1ToL2Messages, emptyTx, newGlobalVariables), - // ); - - // this.log(`Assembled block ${block.number}`, { - // eventName: 'l2-block-built', - // duration: workTimer.ms(), - // publicProcessDuration: publicProcessorDuration, - // rollupCircuitsDuration: rollupCircuitsDuration, - // ...block.getStats(), - // } satisfies L2BlockBuiltStats); + this.log(`Assembled block ${block.number}`, { + eventName: 'l2-block-built', + duration: workTimer.ms(), + publicProcessDuration: publicProcessorDuration, + rollupCircuitsDuration: blockBuildingTimer.ms(), + ...block.getStats(), + } satisfies L2BlockBuiltStats); await assertBlockHeight(); @@ -268,6 +265,7 @@ export class Sequencer { this.log.info(`Submitted rollup block ${block.number} with ${processedValidTxs.length} transactions`); } catch (err) { this.log.error(`Rolling back world state DB due to error assembling block`, (err as any).stack); + this.prover?.cancelBlock(); await this.worldState.getLatest().rollback(); } } @@ -313,44 +311,6 @@ export class Sequencer { return min >= this.lastPublishedBlock; } - /** - * Pads the set of txs to a power of two and assembles a block by calling the block builder. - * @param txs - Processed txs to include in the next block. - * @param l1ToL2Messages - L1 to L2 messages to be part of the block. - * @param emptyTx - Empty tx to repeat at the end of the block to pad to a power of two. - * @param globalVariables - Global variables to use in the block. - * @returns The new block. - */ - protected async buildBlock( - txs: ProcessedTx[], - l1ToL2Messages: Fr[], - emptyTx: ProcessedTx, - globalVariables: GlobalVariables, - ) { - const numRealTxs = txs.length; - const pow2 = Math.log2(numRealTxs); - const totalTxs = 2 ** Math.ceil(pow2); - const blockTicket = await this.prover.startNewBlock( - Math.max(totalTxs, 2), - globalVariables, - l1ToL2Messages, - emptyTx, - ); - - for (const tx of txs) { - await this.prover.addNewTx(tx); - } - - await this.prover.setBlockCompleted(); - - const result = await blockTicket.provingPromise; - if (result.status === PROVING_STATUS.FAILURE) { - throw new Error(`Block proving failed, reason: ${result.reason}`); - } - const blockResult = await this.prover.finaliseBlock(); - return blockResult.block; - } - get coinbase(): EthAddress { return this._coinbase; } From 399b5d44b59d02e67fc7a0815f7f83c9c374b7cf Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 7 Apr 2024 13:05:49 +0100 Subject: [PATCH 14/24] Formatting --- .../sequencer-client/src/sequencer/sequencer.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 01c1e53747ef..750eb757fa0b 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -1,5 +1,5 @@ -import { Tx, type L1ToL2MessageSource, type L2Block, type L2BlockSource, type ProcessedTx } from '@aztec/circuit-types'; -import { PROVING_STATUS, type BlockProver } from '@aztec/circuit-types/interfaces'; +import { type L1ToL2MessageSource, type L2Block, type L2BlockSource, type ProcessedTx, Tx } from '@aztec/circuit-types'; +import { type BlockProver, PROVING_STATUS } from '@aztec/circuit-types/interfaces'; import { type L2BlockBuiltStats } from '@aztec/circuit-types/stats'; import { AztecAddress, EthAddress } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; @@ -209,14 +209,11 @@ export class Sequencer { const pow2 = Math.log2(numRealTxs); const totalTxs = 2 ** Math.ceil(pow2); const blockSize = Math.max(2, totalTxs); - const blockTicket = await this.prover.startNewBlock( - blockSize, - newGlobalVariables, - l1ToL2Messages, - emptyTx, - ); + const blockTicket = await this.prover.startNewBlock(blockSize, newGlobalVariables, l1ToL2Messages, emptyTx); - const [publicProcessorDuration, [processedTxs, failedTxs]] = await elapsed(() => processor.process(validTxs, blockSize, this.prover, txValidator)); + const [publicProcessorDuration, [processedTxs, failedTxs]] = await elapsed(() => + processor.process(validTxs, blockSize, this.prover, txValidator), + ); if (failedTxs.length > 0) { const failedTxData = failedTxs.map(fail => fail.tx); this.log(`Dropping failed txs ${Tx.getHashes(failedTxData).join(', ')}`); From 67a71b54364140a05409ff4a89af00c8460db6d8 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 7 Apr 2024 13:33:05 +0100 Subject: [PATCH 15/24] Tests --- .../src/sequencer/public_processor.test.ts | 65 ++++++++++++++++--- .../src/sequencer/sequencer.test.ts | 5 -- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts b/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts index b1794ec59998..a30c8c42057a 100644 --- a/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts @@ -10,6 +10,7 @@ import { UnencryptedTxL2Logs, mockTx, toTxEffect, + BlockProver, } from '@aztec/circuit-types'; import { ARGS_LENGTH, @@ -57,6 +58,7 @@ describe('public_processor', () => { let publicExecutor: MockProxy; let publicContractsDB: MockProxy; let publicWorldStateDB: MockProxy; + let prover: MockProxy; let proof: Proof; let root: Buffer; @@ -68,6 +70,7 @@ describe('public_processor', () => { publicExecutor = mock(); publicContractsDB = mock(); publicWorldStateDB = mock(); + prover = mock(); proof = makeEmptyProof(); root = Buffer.alloc(32, 5); @@ -98,7 +101,7 @@ describe('public_processor', () => { }); const hash = tx.getTxHash(); - const [processed, failed] = await processor.process([tx]); + const [processed, failed] = await processor.process([tx], 1, prover); expect(processed.length).toBe(1); @@ -126,19 +129,22 @@ describe('public_processor', () => { } expect(failed).toEqual([]); + + expect(prover.addNewTx).toHaveBeenCalledWith(p); }); it('returns failed txs without aborting entire operation', async function () { publicExecutor.simulate.mockRejectedValue(new SimulationError(`Failed`, [])); const tx = mockTx(1, { numberOfNonRevertiblePublicCallRequests: 0, numberOfRevertiblePublicCallRequests: 1 }); - const [processed, failed] = await processor.process([tx]); + const [processed, failed] = await processor.process([tx], 1, prover); expect(processed).toEqual([]); expect(failed[0].tx).toEqual(tx); expect(failed[0].error).toEqual(new SimulationError(`Failed`, [])); expect(publicWorldStateDB.commit).toHaveBeenCalledTimes(0); expect(publicWorldStateDB.rollbackToCommit).toHaveBeenCalledTimes(1); + expect(prover.addNewTx).toHaveBeenCalledTimes(0); }); }); @@ -180,7 +186,7 @@ describe('public_processor', () => { throw new Error(`Unexpected execution request: ${execution}`); }); - const [processed, failed] = await processor.process([tx]); + const [processed, failed] = await processor.process([tx], 1, prover); expect(processed).toHaveLength(1); expect(processed).toEqual([expectedTxByHash(tx)]); @@ -188,6 +194,8 @@ describe('public_processor', () => { expect(publicExecutor.simulate).toHaveBeenCalledTimes(2); expect(publicWorldStateDB.commit).toHaveBeenCalledTimes(1); expect(publicWorldStateDB.rollbackToCommit).toHaveBeenCalledTimes(0); + + expect(prover.addNewTx).toHaveBeenCalledWith(processed[0]); }); it('runs a tx with an enqueued public call with nested execution', async function () { @@ -206,7 +214,7 @@ describe('public_processor', () => { publicExecutor.simulate.mockResolvedValue(publicExecutionResult); - const [processed, failed] = await processor.process([tx]); + const [processed, failed] = await processor.process([tx], 1, prover); expect(processed).toHaveLength(1); expect(processed).toEqual([expectedTxByHash(tx)]); @@ -216,6 +224,39 @@ describe('public_processor', () => { expect(publicWorldStateDB.rollbackToCheckpoint).toHaveBeenCalledTimes(0); expect(publicWorldStateDB.commit).toHaveBeenCalledTimes(1); expect(publicWorldStateDB.rollbackToCommit).toHaveBeenCalledTimes(0); + + expect(prover.addNewTx).toHaveBeenCalledWith(processed[0]); + }); + + it('does not attempt to overfill a block', async function () { + const txs = Array.from([1, 2, 3], (index) => mockTx(index, { numberOfNonRevertiblePublicCallRequests: 0, numberOfRevertiblePublicCallRequests: 1 })); + + let txCount = 0; + + publicExecutor.simulate.mockImplementation(execution => { + const tx = txs[txCount++]; + for (const request of tx.enqueuedPublicFunctionCalls) { + if (execution.contractAddress.equals(request.contractAddress)) { + const result = PublicExecutionResultBuilder.fromPublicCallRequest({ request }).build(); + // result.unencryptedLogs = tx.unencryptedLogs.functionLogs[0]; + return Promise.resolve(result); + } + } + throw new Error(`Unexpected execution request: ${execution}`); + }); + + // We are passing 3 txs but only 2 can fit in the block + const [processed, failed] = await processor.process(txs, 2, prover); + + expect(processed).toHaveLength(2); + expect(processed).toEqual([expectedTxByHash(txs[0]), expectedTxByHash(txs[1])]); + expect(failed).toHaveLength(0); + expect(publicExecutor.simulate).toHaveBeenCalledTimes(2); + expect(publicWorldStateDB.commit).toHaveBeenCalledTimes(2); + expect(publicWorldStateDB.rollbackToCommit).toHaveBeenCalledTimes(0); + + expect(prover.addNewTx).toHaveBeenCalledWith(processed[0]); + expect(prover.addNewTx).toHaveBeenCalledWith(processed[1]); }); it('rolls back app logic db updates on failed public execution, but persists setup/teardown', async function () { @@ -312,7 +353,7 @@ describe('public_processor', () => { const appLogicSpy = jest.spyOn(publicKernel, 'publicKernelCircuitAppLogic'); const teardownSpy = jest.spyOn(publicKernel, 'publicKernelCircuitTeardown'); - const [processed, failed] = await processor.process([tx]); + const [processed, failed] = await processor.process([tx], 1, prover); expect(processed).toHaveLength(1); expect(processed).toEqual([expectedTxByHash(tx)]); @@ -337,6 +378,8 @@ describe('public_processor', () => { ); expect(txEffect.encryptedLogs.getTotalLogCount()).toBe(0); expect(txEffect.unencryptedLogs.getTotalLogCount()).toBe(0); + + expect(prover.addNewTx).toHaveBeenCalledWith(processed[0]); }); it('fails a transaction that reverts in setup', async function () { @@ -425,7 +468,7 @@ describe('public_processor', () => { const appLogicSpy = jest.spyOn(publicKernel, 'publicKernelCircuitAppLogic'); const teardownSpy = jest.spyOn(publicKernel, 'publicKernelCircuitTeardown'); - const [processed, failed] = await processor.process([tx]); + const [processed, failed] = await processor.process([tx], 1, prover); expect(processed).toHaveLength(0); expect(failed).toHaveLength(1); @@ -440,6 +483,8 @@ describe('public_processor', () => { expect(publicWorldStateDB.rollbackToCheckpoint).toHaveBeenCalledTimes(0); expect(publicWorldStateDB.commit).toHaveBeenCalledTimes(0); expect(publicWorldStateDB.rollbackToCommit).toHaveBeenCalledTimes(1); + + expect(prover.addNewTx).toHaveBeenCalledTimes(0); }); it('fails a transaction that reverts in teardown', async function () { @@ -528,7 +573,7 @@ describe('public_processor', () => { const appLogicSpy = jest.spyOn(publicKernel, 'publicKernelCircuitAppLogic'); const teardownSpy = jest.spyOn(publicKernel, 'publicKernelCircuitTeardown'); - const [processed, failed] = await processor.process([tx]); + const [processed, failed] = await processor.process([tx], 1, prover); expect(processed).toHaveLength(0); expect(failed).toHaveLength(1); @@ -542,6 +587,8 @@ describe('public_processor', () => { expect(publicWorldStateDB.rollbackToCheckpoint).toHaveBeenCalledTimes(0); expect(publicWorldStateDB.commit).toHaveBeenCalledTimes(0); expect(publicWorldStateDB.rollbackToCommit).toHaveBeenCalledTimes(1); + + expect(prover.addNewTx).toHaveBeenCalledTimes(0); }); it('runs a tx with setup and teardown phases', async function () { @@ -635,7 +682,7 @@ describe('public_processor', () => { const appLogicSpy = jest.spyOn(publicKernel, 'publicKernelCircuitAppLogic'); const teardownSpy = jest.spyOn(publicKernel, 'publicKernelCircuitTeardown'); - const [processed, failed] = await processor.process([tx]); + const [processed, failed] = await processor.process([tx], 1, prover); expect(processed).toHaveLength(1); expect(processed).toEqual([expectedTxByHash(tx)]); @@ -663,6 +710,8 @@ describe('public_processor', () => { ); expect(txEffect.encryptedLogs.getTotalLogCount()).toBe(0); expect(txEffect.unencryptedLogs.getTotalLogCount()).toBe(0); + + expect(prover.addNewTx).toHaveBeenCalledWith(processed[0]); }); }); }); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index 0a53bfcf5c84..9c72f856637d 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -144,7 +144,6 @@ describe('sequencer', () => { Array(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP).fill(new Fr(0n)), publicProcessor.makeEmptyProcessedTx(), ); - expect(proverClient.addNewTx).toHaveBeenCalledWith(expect.objectContaining({ hash: tx.getTxHash() })); expect(publisher.processL2Block).toHaveBeenCalledWith(block); }); @@ -188,8 +187,6 @@ describe('sequencer', () => { Array(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP).fill(new Fr(0n)), publicProcessor.makeEmptyProcessedTx(), ); - expect(proverClient.addNewTx).toHaveBeenCalledWith(expect.objectContaining({ hash: txs[0].getTxHash() })); - expect(proverClient.addNewTx).toHaveBeenCalledWith(expect.objectContaining({ hash: txs[2].getTxHash() })); expect(publisher.processL2Block).toHaveBeenCalledWith(block); expect(p2p.deleteTxs).toHaveBeenCalledWith([doubleSpendTx.getTxHash()]); }); @@ -229,8 +226,6 @@ describe('sequencer', () => { Array(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP).fill(new Fr(0n)), publicProcessor.makeEmptyProcessedTx(), ); - expect(proverClient.addNewTx).toHaveBeenCalledWith(expect.objectContaining({ hash: txs[0].getTxHash() })); - expect(proverClient.addNewTx).toHaveBeenCalledWith(expect.objectContaining({ hash: txs[2].getTxHash() })); expect(publisher.processL2Block).toHaveBeenCalledWith(block); expect(p2p.deleteTxs).toHaveBeenCalledWith([invalidChainTx.getTxHash()]); }); From 79aa97e6d99c2c2a2f2cbc6cd6b2ec1e3a329740 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 7 Apr 2024 13:49:54 +0100 Subject: [PATCH 16/24] Tests and formatting --- .../src/sequencer/public_processor.test.ts | 33 +++++++++++++++++-- .../src/sequencer/sequencer.test.ts | 4 +++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts b/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts index a30c8c42057a..0e572d0a1f97 100644 --- a/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/public_processor.test.ts @@ -1,4 +1,5 @@ import { + type BlockProver, EncryptedTxL2Logs, type FunctionCall, type ProcessedTx, @@ -10,7 +11,6 @@ import { UnencryptedTxL2Logs, mockTx, toTxEffect, - BlockProver, } from '@aztec/circuit-types'; import { ARGS_LENGTH, @@ -52,6 +52,7 @@ import { type PublicKernelCircuitSimulator } from '../simulator/index.js'; import { type ContractsDataSourcePublicDB, type WorldStatePublicDB } from '../simulator/public_executor.js'; import { RealPublicKernelCircuitSimulator } from '../simulator/public_kernel.js'; import { PublicProcessor } from './public_processor.js'; +import { type TxValidator } from './tx_validator.js'; describe('public_processor', () => { let db: MockProxy; @@ -229,7 +230,9 @@ describe('public_processor', () => { }); it('does not attempt to overfill a block', async function () { - const txs = Array.from([1, 2, 3], (index) => mockTx(index, { numberOfNonRevertiblePublicCallRequests: 0, numberOfRevertiblePublicCallRequests: 1 })); + const txs = Array.from([1, 2, 3], index => + mockTx(index, { numberOfNonRevertiblePublicCallRequests: 0, numberOfRevertiblePublicCallRequests: 1 }), + ); let txCount = 0; @@ -259,6 +262,32 @@ describe('public_processor', () => { expect(prover.addNewTx).toHaveBeenCalledWith(processed[1]); }); + it('does not send a transaction to the prover if validation fails', async function () { + const tx = mockTx(1, { numberOfNonRevertiblePublicCallRequests: 0, numberOfRevertiblePublicCallRequests: 1 }); + + publicExecutor.simulate.mockImplementation(execution => { + for (const request of tx.enqueuedPublicFunctionCalls) { + if (execution.contractAddress.equals(request.contractAddress)) { + const result = PublicExecutionResultBuilder.fromPublicCallRequest({ request }).build(); + // result.unencryptedLogs = tx.unencryptedLogs.functionLogs[0]; + return Promise.resolve(result); + } + } + throw new Error(`Unexpected execution request: ${execution}`); + }); + + const txValidator: MockProxy = mock(); + txValidator.validateTxs.mockRejectedValue([[], [tx]]); + + const [processed, failed] = await processor.process([tx], 1, prover, txValidator); + + expect(processed).toHaveLength(0); + expect(failed).toHaveLength(1); + expect(publicExecutor.simulate).toHaveBeenCalledTimes(1); + + expect(prover.addNewTx).toHaveBeenCalledTimes(0); + }); + it('rolls back app logic db updates on failed public execution, but persists setup/teardown', async function () { const baseContractAddressSeed = 0x200; const baseContractAddress = makeAztecAddress(baseContractAddressSeed); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index 9c72f856637d..fafbf634235f 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -145,6 +145,7 @@ describe('sequencer', () => { publicProcessor.makeEmptyProcessedTx(), ); expect(publisher.processL2Block).toHaveBeenCalledWith(block); + expect(proverClient.cancelBlock).toHaveBeenCalledTimes(0); }); it('builds a block out of several txs rejecting double spends', async () => { @@ -189,6 +190,7 @@ describe('sequencer', () => { ); expect(publisher.processL2Block).toHaveBeenCalledWith(block); expect(p2p.deleteTxs).toHaveBeenCalledWith([doubleSpendTx.getTxHash()]); + expect(proverClient.cancelBlock).toHaveBeenCalledTimes(0); }); it('builds a block out of several txs rejecting incorrect chain ids', async () => { @@ -228,6 +230,7 @@ describe('sequencer', () => { ); expect(publisher.processL2Block).toHaveBeenCalledWith(block); expect(p2p.deleteTxs).toHaveBeenCalledWith([invalidChainTx.getTxHash()]); + expect(proverClient.cancelBlock).toHaveBeenCalledTimes(0); }); it('aborts building a block if the chain moves underneath it', async () => { @@ -263,6 +266,7 @@ describe('sequencer', () => { await sequencer.work(); expect(publisher.processL2Block).not.toHaveBeenCalled(); + expect(proverClient.cancelBlock).toHaveBeenCalledTimes(1); }); }); From f1ac9fb7540451ec50bd20b61ceb87501c3ba96c Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 7 Apr 2024 17:08:23 +0100 Subject: [PATCH 17/24] Fix --- .../circuits.js/src/structs/global_variables.ts | 4 ++++ yarn-project/circuits.js/src/structs/header.ts | 4 ++++ .../prover-client/src/orchestrator/orchestrator.ts | 1 + .../src/sequencer/public_processor.ts | 6 +++++- .../sequencer-client/src/sequencer/sequencer.ts | 12 +++--------- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/yarn-project/circuits.js/src/structs/global_variables.ts b/yarn-project/circuits.js/src/structs/global_variables.ts index d3f762785c4f..32a9100307f3 100644 --- a/yarn-project/circuits.js/src/structs/global_variables.ts +++ b/yarn-project/circuits.js/src/structs/global_variables.ts @@ -106,6 +106,10 @@ export class GlobalVariables { }; } + clone(): GlobalVariables { + return GlobalVariables.fromBuffer(this.toBuffer()); + } + isEmpty(): boolean { return ( this.chainId.isZero() && diff --git a/yarn-project/circuits.js/src/structs/header.ts b/yarn-project/circuits.js/src/structs/header.ts index 0a5413fa5a70..a6639e0f529c 100644 --- a/yarn-project/circuits.js/src/structs/header.ts +++ b/yarn-project/circuits.js/src/structs/header.ts @@ -40,6 +40,10 @@ export class Header { return fields; } + clone(): Header { + return Header.fromBuffer(this.toBuffer()); + } + static fromBuffer(buffer: Buffer | BufferReader): Header { const reader = BufferReader.asReader(buffer); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index 90bd24982908..33d54f86013d 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -236,6 +236,7 @@ export class ProvingOrchestrator { } // we need to pad the rollup with empty transactions + logger.info(`Padding rollup with ${this.provingState.totalNumTxs - this.provingState.transactionsReceived} empty transactions`); for (let i = this.provingState.transactionsReceived; i < this.provingState.totalNumTxs; i++) { const paddingTxIndex = this.provingState.addNewTx(this.provingState.emptyTx); await this.prepareBaseRollupInputs(this.provingState, BigInt(paddingTxIndex), this.provingState!.emptyTx); diff --git a/yarn-project/sequencer-client/src/sequencer/public_processor.ts b/yarn-project/sequencer-client/src/sequencer/public_processor.ts index 9da7db2bc4e8..0d03d43d6d99 100644 --- a/yarn-project/sequencer-client/src/sequencer/public_processor.ts +++ b/yarn-project/sequencer-client/src/sequencer/public_processor.ts @@ -110,6 +110,10 @@ export class PublicProcessor { validateProcessedTx(processedTx); // Re-validate the transaction if (txValidator) { + // Only accept processed transactions that are not double-spends, + // public functions emitting nullifiers would pass earlier check but fail here. + // Note that we're checking all nullifiers generated in the private execution twice, + // we could store the ones already checked and skip them here as an optimization. const [_, invalid] = await txValidator.validateTxs([processedTx]); if (invalid.length) { throw new Error(`Transaction ${invalid[0].hash} invalid after processing public functions`); @@ -142,7 +146,7 @@ export class PublicProcessor { */ public makeEmptyProcessedTx(): ProcessedTx { const { chainId, version } = this.globalVariables; - return makeEmptyProcessedTx(this.historicalHeader, chainId, version); + return makeEmptyProcessedTx(this.historicalHeader.clone(), chainId, version); } private async processTxWithPublicCalls(tx: Tx): Promise<[ProcessedTx, ProcessReturnValues | undefined]> { diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 750eb757fa0b..efe12946b652 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -220,13 +220,7 @@ export class Sequencer { await this.p2pClient.deleteTxs(Tx.getHashes(failedTxData)); } - // Only accept processed transactions that are not double-spends, - // public functions emitting nullifiers would pass earlier check but fail here. - // Note that we're checking all nullifiers generated in the private execution twice, - // we could store the ones already checked and skip them here as an optimization. - const processedValidTxs = processedTxs; - - if (processedValidTxs.length === 0) { + if (processedTxs.length === 0) { this.log('No txs processed correctly to build block. Exiting'); this.prover.cancelBlock(); return; @@ -244,7 +238,7 @@ export class Sequencer { await assertBlockHeight(); // Build the new block by running the rollup circuits - this.log(`Assembling block with txs ${processedValidTxs.map(tx => tx.hash).join(', ')}`); + this.log(`Assembling block with txs ${processedTxs.map(tx => tx.hash).join(', ')}`); await assertBlockHeight(); @@ -259,7 +253,7 @@ export class Sequencer { await assertBlockHeight(); await this.publishL2Block(block); - this.log.info(`Submitted rollup block ${block.number} with ${processedValidTxs.length} transactions`); + this.log.info(`Submitted rollup block ${block.number} with ${processedTxs.length} transactions`); } catch (err) { this.log.error(`Rolling back world state DB due to error assembling block`, (err as any).stack); this.prover?.cancelBlock(); From 25a20b3c30494011a728d4cf5d2e94821378cbbe Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 7 Apr 2024 16:43:42 +0000 Subject: [PATCH 18/24] Comments --- .../src/interfaces/block-prover.ts | 21 ++++++++++++++++++ .../src/orchestrator/orchestrator.ts | 19 ++++++++++++++-- .../prover-client/src/tx-prover/tx-prover.ts | 22 +++++++++++++++++++ .../src/sequencer/sequencer.ts | 14 +++++++----- 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/yarn-project/circuit-types/src/interfaces/block-prover.ts b/yarn-project/circuit-types/src/interfaces/block-prover.ts index 4ba58e123e64..1c1af893e5d4 100644 --- a/yarn-project/circuit-types/src/interfaces/block-prover.ts +++ b/yarn-project/circuit-types/src/interfaces/block-prover.ts @@ -33,6 +33,13 @@ export type BlockResult = { * Provides the ability to generate proofs and build rollups. */ export interface BlockProver { + /** + * Cancels any block that is currently being built and prepares for a new one to be built + * @param numTxs - The complete size of the block, must be a power of 2 + * @param globalVariables - The global variables for this block + * @param l1ToL2Messages - The set of L1 to L2 messages to be included in this block + * @param emptyTx - An instance of an empty transaction to be used in this block + */ startNewBlock( numTxs: number, globalVariables: GlobalVariables, @@ -40,11 +47,25 @@ export interface BlockProver { emptyTx: ProcessedTx, ): Promise; + /** + * Add a processed transaction to the current block + * @param tx - The transaction to be added + */ addNewTx(tx: ProcessedTx): Promise; + /** + * Cancels the block currently being proven. Proofs already bring built may continue but further proofs should not be started. + */ cancelBlock(): void; + /** + * Performs the final archive tree insertion for this block and returns the L2Block and Proof instances + */ finaliseBlock(): Promise; + /** + * Mark the block as having all the transactions it is going to contain. + * Will pad the block to it's complete size with empty transactions and prove all the way to the root rollup. + */ setBlockCompleted(): Promise; } diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index 33d54f86013d..4d3baf6e150d 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -144,7 +144,8 @@ export class ProvingOrchestrator { if (!Number.isInteger(numTxs) || numTxs < 2 || (numTxs & (numTxs - 1)) !== 0) { throw new Error(`Length of txs for the block should be a power of two and at least two (got ${numTxs})`); } - this.provingState?.cancel(); + // Cancel any currently proving block before starting a new one + this.cancelBlock(); logger.info(`Starting new block with ${numTxs} transactions`); // we start the block by enqueueing all of the base parity circuits let baseParityInputs: BaseParityInputs[] = []; @@ -230,23 +231,37 @@ export class ProvingOrchestrator { await this.prepareBaseRollupInputs(this.provingState, BigInt(txIndex), tx); } + /** + * Marks the block as full and pads it to the full power of 2 block size, no more transactions will be accepted. + */ public async setBlockCompleted() { if (!this.provingState) { throw new Error(`Invalid proving state, call startNewBlock before adding transactions or completing the block`); } // we need to pad the rollup with empty transactions - logger.info(`Padding rollup with ${this.provingState.totalNumTxs - this.provingState.transactionsReceived} empty transactions`); + logger.info( + `Padding rollup with ${ + this.provingState.totalNumTxs - this.provingState.transactionsReceived + } empty transactions`, + ); for (let i = this.provingState.transactionsReceived; i < this.provingState.totalNumTxs; i++) { const paddingTxIndex = this.provingState.addNewTx(this.provingState.emptyTx); await this.prepareBaseRollupInputs(this.provingState, BigInt(paddingTxIndex), this.provingState!.emptyTx); } } + /** + * Cancel any further proving of the block + */ public cancelBlock() { this.provingState?.cancel(); } + /** + * Performs the final tree update for the block and returns the fully proven block. + * @returns The fully proven block and proof. + */ public async finaliseBlock() { if (!this.provingState || !this.provingState.rootRollupPublicInputs || !this.provingState.finalProof) { throw new Error(`Invalid proving state, a block must be proven before it can be finalised`); diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index 1b4e25419ba3..61b05ed416f9 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -57,6 +57,13 @@ export class TxProver implements ProverClient { return prover; } + /** + * Cancels any block that is currently being built and prepares for a new one to be built + * @param numTxs - The complete size of the block, must be a power of 2 + * @param globalVariables - The global variables for this block + * @param l1ToL2Messages - The set of L1 to L2 messages to be included in this block + * @param emptyTx - An instance of an empty transaction to be used in this block + */ public async startNewBlock( numTxs: number, globalVariables: GlobalVariables, @@ -68,17 +75,32 @@ export class TxProver implements ProverClient { return this.orchestrator.startNewBlock(numTxs, globalVariables, newL1ToL2Messages, emptyTx); } + /** + * Add a processed transaction to the current block + * @param tx - The transaction to be added + */ public addNewTx(tx: ProcessedTx): Promise { return this.orchestrator.addNewTx(tx); } + /** + * Cancels the block currently being proven. Proofs already bring built may continue but further proofs should not be started. + */ public cancelBlock(): void { this.orchestrator.cancelBlock(); } + + /** + * Performs the final archive tree insertion for this block and returns the L2Block and Proof instances + */ public finaliseBlock(): Promise { return this.orchestrator.finaliseBlock(); } + /** + * Mark the block as having all the transactions it is going to contain. + * Will pad the block to it's complete size with empty transactions and prove all the way to the root rollup. + */ public setBlockCompleted(): Promise { return this.orchestrator.setBlockCompleted(); } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index efe12946b652..03d041060f92 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -205,6 +205,7 @@ export class Sequencer { const blockBuildingTimer = new Timer(); + // We must initialise the block to be a power of 2 in size const numRealTxs = validTxs.length; const pow2 = Math.log2(numRealTxs); const totalTxs = 2 ** Math.ceil(pow2); @@ -226,19 +227,21 @@ export class Sequencer { return; } + await assertBlockHeight(); + + // All real transactions have been added, set the block as full and complete the proving. await this.prover.setBlockCompleted(); const result = await blockTicket.provingPromise; if (result.status === PROVING_STATUS.FAILURE) { throw new Error(`Block proving failed, reason: ${result.reason}`); } - const blockResult = await this.prover.finaliseBlock(); - const block = blockResult.block; await assertBlockHeight(); - // Build the new block by running the rollup circuits - this.log(`Assembling block with txs ${processedTxs.map(tx => tx.hash).join(', ')}`); + // Block is proven, now finalise and publish! + const blockResult = await this.prover.finaliseBlock(); + const block = blockResult.block; await assertBlockHeight(); @@ -250,12 +253,11 @@ export class Sequencer { ...block.getStats(), } satisfies L2BlockBuiltStats); - await assertBlockHeight(); - await this.publishL2Block(block); this.log.info(`Submitted rollup block ${block.number} with ${processedTxs.length} transactions`); } catch (err) { this.log.error(`Rolling back world state DB due to error assembling block`, (err as any).stack); + // Cancel any further proving on the block this.prover?.cancelBlock(); await this.worldState.getLatest().rollback(); } From f852db7d403d5ac939732ee13d95fb0143ec2e72 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 7 Apr 2024 16:43:52 +0000 Subject: [PATCH 19/24] Comments --- cspell.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cspell.json b/cspell.json index f6c12ff5d82e..9180d507c568 100644 --- a/cspell.json +++ b/cspell.json @@ -81,6 +81,8 @@ "fargate", "Fieldeable", "filestat", + "finalise", + "finalised", "flatmap", "foundryup", "frontend", @@ -101,6 +103,7 @@ "herskind", "ierc", "indexeddb", + "initialise", "interruptible", "isequal", "jsons", From e3256a94bf69f7035010f77d95ef9c1caa737959 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 7 Apr 2024 16:45:39 +0000 Subject: [PATCH 20/24] Formatting --- .../src/barretenberg/execution_trace/execution_trace.cpp | 9 +++++---- .../src/barretenberg/execution_trace/execution_trace.hpp | 8 ++++---- .../barretenberg/sumcheck/instance/prover_instance.cpp | 3 ++- .../barretenberg/sumcheck/instance/prover_instance.hpp | 6 ++++-- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp index 5d9fd189d50e..b232fa00799e 100644 --- a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp +++ b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp @@ -53,8 +53,8 @@ void ExecutionTrace_::add_wires_and_selectors_to_proving_key(TraceData& template void ExecutionTrace_::add_memory_records_to_proving_key(TraceData& trace_data, Builder& builder, - typename Flavor::ProvingKey& proving_key) requires - IsUltraPlonkOrHonk + typename Flavor::ProvingKey& proving_key) + requires IsUltraPlonkOrHonk { ASSERT(proving_key.memory_read_records.empty() && proving_key.memory_write_records.empty()); @@ -136,8 +136,9 @@ template void ExecutionTrace_::populate_public_inputs_blo } template -void ExecutionTrace_::add_ecc_op_wires_to_proving_key( - Builder& builder, typename Flavor::ProvingKey& proving_key) requires IsGoblinFlavor +void ExecutionTrace_::add_ecc_op_wires_to_proving_key(Builder& builder, + typename Flavor::ProvingKey& proving_key) + requires IsGoblinFlavor { // Initialize the ecc op wire polynomials to zero on the whole domain std::array op_wire_polynomials; diff --git a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.hpp b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.hpp index 1e487cb1e4ff..3dd3899242e5 100644 --- a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.hpp +++ b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.hpp @@ -70,8 +70,8 @@ template class ExecutionTrace_ { */ static void add_memory_records_to_proving_key(TraceData& trace_data, Builder& builder, - typename Flavor::ProvingKey& proving_key) requires - IsUltraPlonkOrHonk; + typename Flavor::ProvingKey& proving_key) + requires IsUltraPlonkOrHonk; /** * @brief Construct wire polynomials, selector polynomials and copy cycles from raw circuit data @@ -98,8 +98,8 @@ template class ExecutionTrace_ { * @param builder * @param proving_key */ - static void add_ecc_op_wires_to_proving_key(Builder& builder, typename Flavor::ProvingKey& proving_key) requires - IsGoblinFlavor; + static void add_ecc_op_wires_to_proving_key(Builder& builder, typename Flavor::ProvingKey& proving_key) + requires IsGoblinFlavor; }; } // namespace bb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.cpp b/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.cpp index 04f7eedb8a4b..3f0a6b18072f 100644 --- a/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.cpp +++ b/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.cpp @@ -38,7 +38,8 @@ template size_t ProverInstance_::compute_dyadic_size(Circ * @param circuit */ template -void ProverInstance_::construct_databus_polynomials(Circuit& circuit) requires IsGoblinFlavor +void ProverInstance_::construct_databus_polynomials(Circuit& circuit) + requires IsGoblinFlavor { Polynomial public_calldata{ dyadic_circuit_size }; Polynomial calldata_read_counts{ dyadic_circuit_size }; diff --git a/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.hpp b/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.hpp index 696a9ea696b2..d17f45ce5956 100644 --- a/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.hpp +++ b/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.hpp @@ -85,7 +85,8 @@ template class ProverInstance_ { ProverInstance_() = default; ~ProverInstance_() = default; - void compute_databus_id() requires IsGoblinFlavor; + void compute_databus_id() + requires IsGoblinFlavor; private: static constexpr size_t num_zero_rows = Flavor::has_zero_row ? 1 : 0; @@ -94,7 +95,8 @@ template class ProverInstance_ { size_t compute_dyadic_size(Circuit&); - void construct_databus_polynomials(Circuit&) requires IsGoblinFlavor; + void construct_databus_polynomials(Circuit&) + requires IsGoblinFlavor; void construct_table_polynomials(Circuit&, size_t); }; From c3dc0cb314ea186a4469ee06edfeedc41f7df095 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 8 Apr 2024 11:43:40 +0100 Subject: [PATCH 21/24] Comments --- yarn-project/sequencer-client/src/sequencer/sequencer.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index efe12946b652..117cf38e6714 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -226,8 +226,12 @@ export class Sequencer { return; } + // Mark the block as completed. This will pad it with empty transactions enabling it to be fully proven. await this.prover.setBlockCompleted(); + // Here we are now waiting for the block to be proven. + // TODO(@PhilWindle) We should probably periodically check for things like another + // block being published before ours instead of just waiting on our block const result = await blockTicket.provingPromise; if (result.status === PROVING_STATUS.FAILURE) { throw new Error(`Block proving failed, reason: ${result.reason}`); From 6f2affea5ba57edf196564b2ddea14214a7afc5d Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 8 Apr 2024 14:33:35 +0100 Subject: [PATCH 22/24] Attempt fix --- .../end-to-end/src/benchmarks/bench_tx_size_fees.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/yarn-project/end-to-end/src/benchmarks/bench_tx_size_fees.test.ts b/yarn-project/end-to-end/src/benchmarks/bench_tx_size_fees.test.ts index def00fbbb70c..c79c504aa452 100644 --- a/yarn-project/end-to-end/src/benchmarks/bench_tx_size_fees.test.ts +++ b/yarn-project/end-to-end/src/benchmarks/bench_tx_size_fees.test.ts @@ -56,11 +56,10 @@ describe('benchmarks/tx_size_fees', () => { beforeAll(async () => { await Promise.all([ gas.methods.mint_public(aliceWallet.getAddress(), 1000n).send().wait(), - token.methods.privately_mint_private_note(1000n).send().wait(), - token.methods.mint_public(aliceWallet.getAddress(), 1000n).send().wait(), - gas.methods.mint_public(fpc.address, 1000n).send().wait(), ]); + await token.methods.privately_mint_private_note(1000n).send().wait(); + await token.methods.mint_public(aliceWallet.getAddress(), 1000n).send().wait(); }); it.each<() => Promise>([ From 9e8f2a5639a9402539883bc10026b6b7ddc5704d Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 8 Apr 2024 13:38:48 +0000 Subject: [PATCH 23/24] Formatting --- .../src/barretenberg/execution_trace/execution_trace.cpp | 9 +++++---- .../src/barretenberg/execution_trace/execution_trace.hpp | 8 ++++---- .../barretenberg/sumcheck/instance/prover_instance.cpp | 3 ++- .../barretenberg/sumcheck/instance/prover_instance.hpp | 6 ++++-- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp index 5d9fd189d50e..b232fa00799e 100644 --- a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp +++ b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp @@ -53,8 +53,8 @@ void ExecutionTrace_::add_wires_and_selectors_to_proving_key(TraceData& template void ExecutionTrace_::add_memory_records_to_proving_key(TraceData& trace_data, Builder& builder, - typename Flavor::ProvingKey& proving_key) requires - IsUltraPlonkOrHonk + typename Flavor::ProvingKey& proving_key) + requires IsUltraPlonkOrHonk { ASSERT(proving_key.memory_read_records.empty() && proving_key.memory_write_records.empty()); @@ -136,8 +136,9 @@ template void ExecutionTrace_::populate_public_inputs_blo } template -void ExecutionTrace_::add_ecc_op_wires_to_proving_key( - Builder& builder, typename Flavor::ProvingKey& proving_key) requires IsGoblinFlavor +void ExecutionTrace_::add_ecc_op_wires_to_proving_key(Builder& builder, + typename Flavor::ProvingKey& proving_key) + requires IsGoblinFlavor { // Initialize the ecc op wire polynomials to zero on the whole domain std::array op_wire_polynomials; diff --git a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.hpp b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.hpp index 1e487cb1e4ff..3dd3899242e5 100644 --- a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.hpp +++ b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.hpp @@ -70,8 +70,8 @@ template class ExecutionTrace_ { */ static void add_memory_records_to_proving_key(TraceData& trace_data, Builder& builder, - typename Flavor::ProvingKey& proving_key) requires - IsUltraPlonkOrHonk; + typename Flavor::ProvingKey& proving_key) + requires IsUltraPlonkOrHonk; /** * @brief Construct wire polynomials, selector polynomials and copy cycles from raw circuit data @@ -98,8 +98,8 @@ template class ExecutionTrace_ { * @param builder * @param proving_key */ - static void add_ecc_op_wires_to_proving_key(Builder& builder, typename Flavor::ProvingKey& proving_key) requires - IsGoblinFlavor; + static void add_ecc_op_wires_to_proving_key(Builder& builder, typename Flavor::ProvingKey& proving_key) + requires IsGoblinFlavor; }; } // namespace bb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.cpp b/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.cpp index 04f7eedb8a4b..3f0a6b18072f 100644 --- a/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.cpp +++ b/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.cpp @@ -38,7 +38,8 @@ template size_t ProverInstance_::compute_dyadic_size(Circ * @param circuit */ template -void ProverInstance_::construct_databus_polynomials(Circuit& circuit) requires IsGoblinFlavor +void ProverInstance_::construct_databus_polynomials(Circuit& circuit) + requires IsGoblinFlavor { Polynomial public_calldata{ dyadic_circuit_size }; Polynomial calldata_read_counts{ dyadic_circuit_size }; diff --git a/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.hpp b/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.hpp index 696a9ea696b2..d17f45ce5956 100644 --- a/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.hpp +++ b/barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.hpp @@ -85,7 +85,8 @@ template class ProverInstance_ { ProverInstance_() = default; ~ProverInstance_() = default; - void compute_databus_id() requires IsGoblinFlavor; + void compute_databus_id() + requires IsGoblinFlavor; private: static constexpr size_t num_zero_rows = Flavor::has_zero_row ? 1 : 0; @@ -94,7 +95,8 @@ template class ProverInstance_ { size_t compute_dyadic_size(Circuit&); - void construct_databus_polynomials(Circuit&) requires IsGoblinFlavor; + void construct_databus_polynomials(Circuit&) + requires IsGoblinFlavor; void construct_table_polynomials(Circuit&, size_t); }; From a0ae521ef5f84be58ac8d45883676eed9f9dd5da Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 8 Apr 2024 14:01:58 +0000 Subject: [PATCH 24/24] Formatting --- yarn-project/sequencer-client/src/sequencer/sequencer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 72fc65b8c74e..e400e3ba9356 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -232,7 +232,7 @@ export class Sequencer { // All real transactions have been added, set the block as full and complete the proving. await this.prover.setBlockCompleted(); - // Here we are now waiting for the block to be proven. + // Here we are now waiting for the block to be proven. // TODO(@PhilWindle) We should probably periodically check for things like another // block being published before ours instead of just waiting on our block const result = await blockTicket.provingPromise;