diff --git a/yarn-project/archiver/src/modules/data_source_base.ts b/yarn-project/archiver/src/modules/data_source_base.ts index 2cd512830e74..7a8cfc85f238 100644 --- a/yarn-project/archiver/src/modules/data_source_base.ts +++ b/yarn-project/archiver/src/modules/data_source_base.ts @@ -1,18 +1,12 @@ +import { range } from '@aztec/foundation/array'; import { BlockNumber, CheckpointNumber, type EpochNumber, type SlotNumber } from '@aztec/foundation/branded-types'; import type { Fr } from '@aztec/foundation/curves/bn254'; import type { EthAddress } from '@aztec/foundation/eth-address'; import { isDefined } from '@aztec/foundation/types'; import type { FunctionSelector } from '@aztec/stdlib/abi'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; -import { - type BlockData, - type BlockHash, - CheckpointedL2Block, - CommitteeAttestation, - L2Block, - type L2Tips, -} from '@aztec/stdlib/block'; -import { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import { type BlockData, type BlockHash, CheckpointedL2Block, L2Block, type L2Tips } from '@aztec/stdlib/block'; +import { Checkpoint, type CheckpointData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { ContractClassPublic, ContractDataSource, ContractInstanceWithAddress } from '@aztec/stdlib/contract'; import { type L1RollupConstants, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers'; import type { GetContractClassLogsResponse, GetPublicLogsResponse } from '@aztec/stdlib/interfaces/client'; @@ -24,7 +18,6 @@ import type { BlockHeader, IndexedTxEffect, TxHash, TxReceipt } from '@aztec/std import type { UInt64 } from '@aztec/stdlib/types'; import type { ArchiverDataSource } from '../interfaces.js'; -import type { CheckpointData } from '../store/block_store.js'; import type { KVArchiverDataStore } from '../store/kv_archiver_store.js'; import type { ValidateCheckpointResult } from './validation.js'; @@ -121,7 +114,7 @@ export abstract class ArchiverDataSourceBase if (!checkpointData) { return undefined; } - return BlockNumber(checkpointData.startBlock + checkpointData.numBlocks - 1); + return BlockNumber(checkpointData.startBlock + checkpointData.blockCount - 1); } public getCheckpointedBlocks(from: BlockNumber, limit: number): Promise { @@ -238,28 +231,21 @@ export abstract class ArchiverDataSourceBase public async getCheckpoints(checkpointNumber: CheckpointNumber, limit: number): Promise { const checkpoints = await this.store.getRangeOfCheckpoints(checkpointNumber, limit); - const blocks = ( - await Promise.all(checkpoints.map(ch => this.store.getBlocksForCheckpoint(ch.checkpointNumber))) - ).filter(isDefined); - - const fullCheckpoints: PublishedCheckpoint[] = []; - for (let i = 0; i < checkpoints.length; i++) { - const blocksForCheckpoint = blocks[i]; - const checkpoint = checkpoints[i]; - const fullCheckpoint = new Checkpoint( - checkpoint.archive, - checkpoint.header, - blocksForCheckpoint, - checkpoint.checkpointNumber, - ); - const publishedCheckpoint = new PublishedCheckpoint( - fullCheckpoint, - checkpoint.l1, - checkpoint.attestations.map(x => CommitteeAttestation.fromBuffer(x)), - ); - fullCheckpoints.push(publishedCheckpoint); + return Promise.all(checkpoints.map(ch => this.getPublishedCheckpointFromCheckpointData(ch))); + } + + private async getPublishedCheckpointFromCheckpointData(checkpoint: CheckpointData): Promise { + const blocksForCheckpoint = await this.store.getBlocksForCheckpoint(checkpoint.checkpointNumber); + if (!blocksForCheckpoint) { + throw new Error(`Blocks for checkpoint ${checkpoint.checkpointNumber} not found`); } - return fullCheckpoints; + const fullCheckpoint = new Checkpoint( + checkpoint.archive, + checkpoint.header, + blocksForCheckpoint, + checkpoint.checkpointNumber, + ); + return new PublishedCheckpoint(fullCheckpoint, checkpoint.l1, checkpoint.attestations); } public getBlocksForSlot(slotNumber: SlotNumber): Promise { @@ -267,84 +253,44 @@ export abstract class ArchiverDataSourceBase } public async getCheckpointedBlocksForEpoch(epochNumber: EpochNumber): Promise { - if (!this.l1Constants) { - throw new Error('L1 constants not set'); - } - - const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1Constants); - const blocks: CheckpointedL2Block[] = []; - - // Walk the list of checkpoints backwards and filter by slots matching the requested epoch. - // We'll typically ask for checkpoints for a very recent epoch, so we shouldn't need an index here. - let checkpoint = await this.store.getCheckpointData(await this.store.getSynchedCheckpointNumber()); - const slot = (b: CheckpointData) => b.header.slotNumber; - while (checkpoint && slot(checkpoint) >= start) { - if (slot(checkpoint) <= end) { - // push the blocks on backwards - const endBlock = checkpoint.startBlock + checkpoint.numBlocks - 1; - for (let i = endBlock; i >= checkpoint.startBlock; i--) { - const checkpointedBlock = await this.getCheckpointedBlock(BlockNumber(i)); - if (checkpointedBlock) { - blocks.push(checkpointedBlock); - } - } - } - checkpoint = await this.store.getCheckpointData(CheckpointNumber(checkpoint.checkpointNumber - 1)); - } - - return blocks.reverse(); + const checkpointsData = await this.getCheckpointsDataForEpoch(epochNumber); + const blocks = await Promise.all( + checkpointsData.flatMap(checkpoint => + range(checkpoint.blockCount, checkpoint.startBlock).map(blockNumber => + this.getCheckpointedBlock(BlockNumber(blockNumber)), + ), + ), + ); + return blocks.filter(isDefined); } public async getCheckpointedBlockHeadersForEpoch(epochNumber: EpochNumber): Promise { - if (!this.l1Constants) { - throw new Error('L1 constants not set'); - } - - const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1Constants); - const blocks: BlockHeader[] = []; - - // Walk the list of checkpoints backwards and filter by slots matching the requested epoch. - // We'll typically ask for checkpoints for a very recent epoch, so we shouldn't need an index here. - let checkpoint = await this.store.getCheckpointData(await this.store.getSynchedCheckpointNumber()); - const slot = (b: CheckpointData) => b.header.slotNumber; - while (checkpoint && slot(checkpoint) >= start) { - if (slot(checkpoint) <= end) { - // push the blocks on backwards - const endBlock = checkpoint.startBlock + checkpoint.numBlocks - 1; - for (let i = endBlock; i >= checkpoint.startBlock; i--) { - const block = await this.getBlockHeader(BlockNumber(i)); - if (block) { - blocks.push(block); - } - } - } - checkpoint = await this.store.getCheckpointData(CheckpointNumber(checkpoint.checkpointNumber - 1)); - } - return blocks.reverse(); + const checkpointsData = await this.getCheckpointsDataForEpoch(epochNumber); + const blocks = await Promise.all( + checkpointsData.flatMap(checkpoint => + range(checkpoint.blockCount, checkpoint.startBlock).map(blockNumber => + this.getBlockHeader(BlockNumber(blockNumber)), + ), + ), + ); + return blocks.filter(isDefined); } public async getCheckpointsForEpoch(epochNumber: EpochNumber): Promise { + const checkpointsData = await this.getCheckpointsDataForEpoch(epochNumber); + return Promise.all( + checkpointsData.map(data => this.getPublishedCheckpointFromCheckpointData(data).then(p => p.checkpoint)), + ); + } + + /** Returns checkpoint data for all checkpoints whose slot falls within the given epoch. */ + public getCheckpointsDataForEpoch(epochNumber: EpochNumber): Promise { if (!this.l1Constants) { throw new Error('L1 constants not set'); } const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1Constants); - const checkpoints: Checkpoint[] = []; - - // Walk the list of checkpoints backwards and filter by slots matching the requested epoch. - // We'll typically ask for checkpoints for a very recent epoch, so we shouldn't need an index here. - let checkpointData = await this.store.getCheckpointData(await this.store.getSynchedCheckpointNumber()); - const slot = (b: CheckpointData) => b.header.slotNumber; - while (checkpointData && slot(checkpointData) >= start) { - if (slot(checkpointData) <= end) { - // push the checkpoints on backwards - const [checkpoint] = await this.getCheckpoints(checkpointData.checkpointNumber, 1); - checkpoints.push(checkpoint.checkpoint); - } - checkpointData = await this.store.getCheckpointData(CheckpointNumber(checkpointData.checkpointNumber - 1)); - } - - return checkpoints.reverse(); + return this.store.getCheckpointDataForSlotRange(start, end); } public async getBlock(number: BlockNumber): Promise { diff --git a/yarn-project/archiver/src/modules/instrumentation.ts b/yarn-project/archiver/src/modules/instrumentation.ts index f0a18a2d17d7..fbf91cf16a1a 100644 --- a/yarn-project/archiver/src/modules/instrumentation.ts +++ b/yarn-project/archiver/src/modules/instrumentation.ts @@ -1,5 +1,6 @@ import { createLogger } from '@aztec/foundation/log'; import type { L2Block } from '@aztec/stdlib/block'; +import type { CheckpointData } from '@aztec/stdlib/checkpoint'; import { Attributes, type Gauge, @@ -13,8 +14,6 @@ import { createUpDownCounterWithDefault, } from '@aztec/telemetry-client'; -import type { CheckpointData } from '../store/block_store.js'; - export class ArchiverInstrumentation { public readonly tracer: Tracer; @@ -134,7 +133,7 @@ export class ArchiverInstrumentation { } public updateLastProvenCheckpoint(checkpoint: CheckpointData) { - const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.numBlocks - 1; + const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.blockCount - 1; this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' }); this.checkpointHeight.record(checkpoint.checkpointNumber, { [Attributes.STATUS]: 'proven' }); } diff --git a/yarn-project/archiver/src/modules/l1_synchronizer.ts b/yarn-project/archiver/src/modules/l1_synchronizer.ts index bc73f3bb35c2..22b1ed5aba29 100644 --- a/yarn-project/archiver/src/modules/l1_synchronizer.ts +++ b/yarn-project/archiver/src/modules/l1_synchronizer.ts @@ -590,7 +590,7 @@ export class ArchiverL1Synchronizer implements Traceable { const provenEpochNumber: EpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants); const lastBlockNumberInCheckpoint = localCheckpointForDestinationProvenCheckpointNumber.startBlock + - localCheckpointForDestinationProvenCheckpointNumber.numBlocks - + localCheckpointForDestinationProvenCheckpointNumber.blockCount - 1; this.events.emit(L2BlockSourceEvents.L2BlockProven, { diff --git a/yarn-project/archiver/src/store/block_store.ts b/yarn-project/archiver/src/store/block_store.ts index 636df8cc299a..a9ec9a501c85 100644 --- a/yarn-project/archiver/src/store/block_store.ts +++ b/yarn-project/archiver/src/store/block_store.ts @@ -19,7 +19,7 @@ import { deserializeValidateCheckpointResult, serializeValidateCheckpointResult, } from '@aztec/stdlib/block'; -import { L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import { type CheckpointData, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { CheckpointHeader } from '@aztec/stdlib/rollup'; import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees'; @@ -62,23 +62,14 @@ type BlockStorage = { type CheckpointStorage = { header: Buffer; archive: Buffer; + checkpointOutHash: Buffer; checkpointNumber: number; startBlock: number; - numBlocks: number; + blockCount: number; l1: Buffer; attestations: Buffer[]; }; -export type CheckpointData = { - checkpointNumber: CheckpointNumber; - header: CheckpointHeader; - archive: AppendOnlyTreeSnapshot; - startBlock: number; - numBlocks: number; - l1: L1PublishedData; - attestations: Buffer[]; -}; - export type RemoveCheckpointsResult = { blocksRemoved: L2Block[] | undefined }; /** @@ -91,6 +82,9 @@ export class BlockStore { /** Map checkpoint number to checkpoint data */ #checkpoints: AztecAsyncMap; + /** Map slot number to checkpoint number, for looking up checkpoints by slot range. */ + #slotToCheckpoint: AztecAsyncMap; + /** Map block hash to list of tx hashes */ #blockTxs: AztecAsyncMap; @@ -131,6 +125,7 @@ export class BlockStore { this.#lastProvenCheckpoint = db.openSingleton('archiver_last_proven_l2_checkpoint'); this.#pendingChainValidationStatus = db.openSingleton('archiver_pending_chain_validation_status'); this.#checkpoints = db.openMap('archiver_checkpoints'); + this.#slotToCheckpoint = db.openMap('archiver_slot_to_checkpoint'); } /** @@ -274,7 +269,7 @@ export class BlockStore { // If we have a previous checkpoint then we need to get the previous block number if (previousCheckpointData !== undefined) { - previousBlockNumber = BlockNumber(previousCheckpointData.startBlock + previousCheckpointData.numBlocks - 1); + previousBlockNumber = BlockNumber(previousCheckpointData.startBlock + previousCheckpointData.blockCount - 1); previousBlock = await this.getBlock(previousBlockNumber); if (previousBlock === undefined) { // We should be able to get the required previous block @@ -338,12 +333,16 @@ export class BlockStore { await this.#checkpoints.set(checkpoint.checkpoint.number, { header: checkpoint.checkpoint.header.toBuffer(), archive: checkpoint.checkpoint.archive.toBuffer(), + checkpointOutHash: checkpoint.checkpoint.getCheckpointOutHash().toBuffer(), l1: checkpoint.l1.toBuffer(), attestations: checkpoint.attestations.map(attestation => attestation.toBuffer()), checkpointNumber: checkpoint.checkpoint.number, startBlock: checkpoint.checkpoint.blocks[0].number, - numBlocks: checkpoint.checkpoint.blocks.length, + blockCount: checkpoint.checkpoint.blocks.length, }); + + // Update slot-to-checkpoint index + await this.#slotToCheckpoint.set(checkpoint.checkpoint.header.slotNumber, checkpoint.checkpoint.number); } await this.#lastSynchedL1Block.set(checkpoints[checkpoints.length - 1].l1.blockNumber); @@ -426,7 +425,7 @@ export class BlockStore { if (!targetCheckpoint) { throw new Error(`Target checkpoint ${checkpointNumber} not found in store`); } - lastBlockToKeep = BlockNumber(targetCheckpoint.startBlock + targetCheckpoint.numBlocks - 1); + lastBlockToKeep = BlockNumber(targetCheckpoint.startBlock + targetCheckpoint.blockCount - 1); } // Remove all blocks after lastBlockToKeep (both checkpointed and uncheckpointed) @@ -434,6 +433,11 @@ export class BlockStore { // Remove all checkpoints after the target for (let c = latestCheckpointNumber; c > checkpointNumber; c = CheckpointNumber(c - 1)) { + const checkpointStorage = await this.#checkpoints.getAsync(c); + if (checkpointStorage) { + const slotNumber = CheckpointHeader.fromBuffer(checkpointStorage.header).slotNumber; + await this.#slotToCheckpoint.delete(slotNumber); + } await this.#checkpoints.delete(c); this.#log.debug(`Removed checkpoint ${c}`); } @@ -462,17 +466,32 @@ export class BlockStore { return checkpoints; } - private checkpointDataFromCheckpointStorage(checkpointStorage: CheckpointStorage) { - const data: CheckpointData = { + /** Returns checkpoint data for all checkpoints whose slot falls within the given range (inclusive). */ + async getCheckpointDataForSlotRange(startSlot: SlotNumber, endSlot: SlotNumber): Promise { + const result: CheckpointData[] = []; + for await (const [, checkpointNumber] of this.#slotToCheckpoint.entriesAsync({ + start: startSlot, + end: endSlot + 1, + })) { + const checkpointStorage = await this.#checkpoints.getAsync(checkpointNumber); + if (checkpointStorage) { + result.push(this.checkpointDataFromCheckpointStorage(checkpointStorage)); + } + } + return result; + } + + private checkpointDataFromCheckpointStorage(checkpointStorage: CheckpointStorage): CheckpointData { + return { header: CheckpointHeader.fromBuffer(checkpointStorage.header), archive: AppendOnlyTreeSnapshot.fromBuffer(checkpointStorage.archive), + checkpointOutHash: Fr.fromBuffer(checkpointStorage.checkpointOutHash), checkpointNumber: CheckpointNumber(checkpointStorage.checkpointNumber), - startBlock: checkpointStorage.startBlock, - numBlocks: checkpointStorage.numBlocks, + startBlock: BlockNumber(checkpointStorage.startBlock), + blockCount: checkpointStorage.blockCount, l1: L1PublishedData.fromBuffer(checkpointStorage.l1), - attestations: checkpointStorage.attestations, + attestations: checkpointStorage.attestations.map(buf => CommitteeAttestation.fromBuffer(buf)), }; - return data; } async getBlocksForCheckpoint(checkpointNumber: CheckpointNumber): Promise { @@ -484,7 +503,7 @@ export class BlockStore { const blocksForCheckpoint = await toArray( this.#blocks.entriesAsync({ start: checkpoint.startBlock, - end: checkpoint.startBlock + checkpoint.numBlocks, + end: checkpoint.startBlock + checkpoint.blockCount, }), ); @@ -557,7 +576,7 @@ export class BlockStore { if (!checkpointStorage) { throw new CheckpointNotFoundError(provenCheckpointNumber); } else { - return BlockNumber(checkpointStorage.startBlock + checkpointStorage.numBlocks - 1); + return BlockNumber(checkpointStorage.startBlock + checkpointStorage.blockCount - 1); } } @@ -922,7 +941,7 @@ export class BlockStore { if (!checkpoint) { return BlockNumber(INITIAL_L2_BLOCK_NUM - 1); } - return BlockNumber(checkpoint.startBlock + checkpoint.numBlocks - 1); + return BlockNumber(checkpoint.startBlock + checkpoint.blockCount - 1); } async getLatestL2BlockNumber(): Promise { diff --git a/yarn-project/archiver/src/store/kv_archiver_store.test.ts b/yarn-project/archiver/src/store/kv_archiver_store.test.ts index f0100cc2cce0..d05044ded8d2 100644 --- a/yarn-project/archiver/src/store/kv_archiver_store.test.ts +++ b/yarn-project/archiver/src/store/kv_archiver_store.test.ts @@ -1474,7 +1474,7 @@ describe('KVArchiverDataStore', () => { expect(checkpoints.length).toBe(1); expect(checkpoints[0].checkpointNumber).toBe(1); expect(checkpoints[0].startBlock).toBe(1); - expect(checkpoints[0].numBlocks).toBe(2); + expect(checkpoints[0].blockCount).toBe(2); }); it('returns multiple checkpoints in order', async () => { @@ -1504,7 +1504,7 @@ describe('KVArchiverDataStore', () => { expect(checkpoints.length).toBe(3); expect(checkpoints.map(c => c.checkpointNumber)).toEqual([1, 2, 3]); expect(checkpoints.map(c => c.startBlock)).toEqual([1, 3, 6]); - expect(checkpoints.map(c => c.numBlocks)).toEqual([2, 3, 1]); + expect(checkpoints.map(c => c.blockCount)).toEqual([2, 3, 1]); }); it('respects the from parameter', async () => { @@ -1586,7 +1586,7 @@ describe('KVArchiverDataStore', () => { const data = checkpoints[0]; expect(data.checkpointNumber).toBe(1); expect(data.startBlock).toBe(1); - expect(data.numBlocks).toBe(3); + expect(data.blockCount).toBe(3); expect(data.l1.blockNumber).toBe(42n); expect(data.header.equals(checkpoint.checkpoint.header)).toBe(true); expect(data.archive.equals(checkpoint.checkpoint.archive)).toBe(true); diff --git a/yarn-project/archiver/src/store/kv_archiver_store.ts b/yarn-project/archiver/src/store/kv_archiver_store.ts index 24447c10ae5a..d46075e2a588 100644 --- a/yarn-project/archiver/src/store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/store/kv_archiver_store.ts @@ -13,7 +13,7 @@ import { L2Block, type ValidateCheckpointResult, } from '@aztec/stdlib/block'; -import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import type { CheckpointData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { ContractClassPublic, ContractDataSource, @@ -31,7 +31,7 @@ import type { UInt64 } from '@aztec/stdlib/types'; import { join } from 'path'; import type { InboxMessage } from '../structs/inbox_message.js'; -import { BlockStore, type CheckpointData, type RemoveCheckpointsResult } from './block_store.js'; +import { BlockStore, type RemoveCheckpointsResult } from './block_store.js'; import { ContractClassStore } from './contract_class_store.js'; import { ContractInstanceStore } from './contract_instance_store.js'; import { LogStore } from './log_store.js'; @@ -645,6 +645,11 @@ export class KVArchiverDataStore implements ContractDataSource { return this.#blockStore.getCheckpointData(checkpointNumber); } + /** Returns checkpoint data for all checkpoints whose slot falls within the given range (inclusive). */ + getCheckpointDataForSlotRange(startSlot: SlotNumber, endSlot: SlotNumber): Promise { + return this.#blockStore.getCheckpointDataForSlotRange(startSlot, endSlot); + } + /** * Gets all blocks that have the given slot number. * @param slotNumber - The slot number to search for. diff --git a/yarn-project/archiver/src/test/mock_archiver.ts b/yarn-project/archiver/src/test/mock_archiver.ts index a613dabf011b..bcdcc3928d96 100644 --- a/yarn-project/archiver/src/test/mock_archiver.ts +++ b/yarn-project/archiver/src/test/mock_archiver.ts @@ -56,8 +56,9 @@ export class MockPrefilledArchiver extends MockArchiver { } const fromBlock = this.l2Blocks.length; - // TODO: Add L2 blocks and checkpoints separately once archiver has the apis for that. - this.addProposedBlocks(this.prefilled.slice(fromBlock, fromBlock + numBlocks).flatMap(c => c.blocks)); + const checkpointsToAdd = this.prefilled.slice(fromBlock, fromBlock + numBlocks); + this.addProposedBlocks(checkpointsToAdd.flatMap(c => c.blocks)); + this.checkpointList.push(...checkpointsToAdd); return Promise.resolve(); } } diff --git a/yarn-project/archiver/src/test/mock_l2_block_source.ts b/yarn-project/archiver/src/test/mock_l2_block_source.ts index d0a44894ec4e..ff4a0fe4af52 100644 --- a/yarn-project/archiver/src/test/mock_l2_block_source.ts +++ b/yarn-project/archiver/src/test/mock_l2_block_source.ts @@ -16,9 +16,11 @@ import { type L2Tips, type ValidateCheckpointResult, } from '@aztec/stdlib/block'; -import { Checkpoint, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import { Checkpoint, type CheckpointData, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { ContractClassPublic, ContractDataSource, ContractInstanceWithAddress } from '@aztec/stdlib/contract'; import { EmptyL1RollupConstants, type L1RollupConstants, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers'; +import { computeCheckpointOutHash } from '@aztec/stdlib/messaging'; +import { CheckpointHeader } from '@aztec/stdlib/rollup'; import { type BlockHeader, TxExecutionResult, TxHash, TxReceipt, TxStatus } from '@aztec/stdlib/tx'; import type { UInt64 } from '@aztec/stdlib/types'; @@ -27,6 +29,7 @@ import type { UInt64 } from '@aztec/stdlib/types'; */ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { protected l2Blocks: L2Block[] = []; + protected checkpointList: Checkpoint[] = []; private provenBlockNumber: number = 0; private finalizedBlockNumber: number = 0; @@ -34,14 +37,30 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { private log = createLogger('archiver:mock_l2_block_source'); + /** Creates blocks grouped into single-block checkpoints. */ public async createBlocks(numBlocks: number) { - for (let i = 0; i < numBlocks; i++) { - const blockNum = this.l2Blocks.length + 1; - const block = await L2Block.random(BlockNumber(blockNum), { slotNumber: SlotNumber(blockNum) }); - this.l2Blocks.push(block); + await this.createCheckpoints(numBlocks, 1); + } + + /** Creates checkpoints, each containing `blocksPerCheckpoint` blocks. */ + public async createCheckpoints(numCheckpoints: number, blocksPerCheckpoint: number = 1) { + for (let c = 0; c < numCheckpoints; c++) { + const checkpointNum = CheckpointNumber(this.checkpointList.length + 1); + const startBlockNum = this.l2Blocks.length + 1; + const slotNumber = SlotNumber(Number(checkpointNum)); + const checkpoint = await Checkpoint.random(checkpointNum, { + numBlocks: blocksPerCheckpoint, + startBlockNumber: startBlockNum, + slotNumber, + checkpointNumber: checkpointNum, + }); + this.checkpointList.push(checkpoint); + this.l2Blocks.push(...checkpoint.blocks); } - this.log.verbose(`Created ${numBlocks} blocks in the mock L2 block source`); + this.log.verbose( + `Created ${numCheckpoints} checkpoints with ${blocksPerCheckpoint} blocks each in the mock L2 block source`, + ); } public addProposedBlocks(blocks: L2Block[]) { @@ -51,6 +70,16 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { public removeBlocks(numBlocks: number) { this.l2Blocks = this.l2Blocks.slice(0, -numBlocks); + const maxBlockNum = this.l2Blocks.length; + // Remove any checkpoint whose last block is beyond the remaining blocks. + this.checkpointList = this.checkpointList.filter(c => { + const lastBlockNum = c.blocks[0].number + c.blocks.length - 1; + return lastBlockNum <= maxBlockNum; + }); + // Keep tip numbers consistent with remaining blocks. + this.checkpointedBlockNumber = Math.min(this.checkpointedBlockNumber, maxBlockNum); + this.provenBlockNumber = Math.min(this.provenBlockNumber, maxBlockNum); + this.finalizedBlockNumber = Math.min(this.finalizedBlockNumber, maxBlockNum); this.log.verbose(`Removed ${numBlocks} blocks from the mock L2 block source`); } @@ -66,7 +95,33 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { } public setCheckpointedBlockNumber(checkpointedBlockNumber: number) { + const prevCheckpointed = this.checkpointedBlockNumber; this.checkpointedBlockNumber = checkpointedBlockNumber; + // Auto-create single-block checkpoints for newly checkpointed blocks that don't have one yet. + // This handles blocks added via addProposedBlocks that are now being marked as checkpointed. + const newCheckpoints: Checkpoint[] = []; + for (let blockNum = prevCheckpointed + 1; blockNum <= checkpointedBlockNumber; blockNum++) { + const block = this.l2Blocks[blockNum - 1]; + if (!block) { + continue; + } + if (this.checkpointList.some(c => c.blocks.some(b => b.number === block.number))) { + continue; + } + const checkpointNum = CheckpointNumber(this.checkpointList.length + newCheckpoints.length + 1); + const checkpoint = new Checkpoint( + block.archive, + CheckpointHeader.random({ slotNumber: block.header.globalVariables.slotNumber }), + [block], + checkpointNum, + ); + newCheckpoints.push(checkpoint); + } + // Insert new checkpoints in order by number. + if (newCheckpoints.length > 0) { + this.checkpointList.push(...newCheckpoints); + this.checkpointList.sort((a, b) => a.number - b.number); + } } /** @@ -113,13 +168,7 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { if (!block) { return Promise.resolve(undefined); } - const checkpointedBlock = new CheckpointedL2Block( - CheckpointNumber.fromBlockNumber(number), - block, - new L1PublishedData(BigInt(number), BigInt(number), `0x${number.toString(16).padStart(64, '0')}`), - [], - ); - return Promise.resolve(checkpointedBlock); + return Promise.resolve(this.toCheckpointedBlock(block)); } public async getCheckpointedBlocks(from: BlockNumber, limit: number): Promise { @@ -168,44 +217,22 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { } public getCheckpoints(from: CheckpointNumber, limit: number) { - // TODO(mbps): Implement this properly. This only works when we have one block per checkpoint. - const blocks = this.l2Blocks.slice(from - 1, from - 1 + limit); - return Promise.all( - blocks.map(async block => { - // Create a checkpoint from the block - manually construct since L2Block doesn't have toCheckpoint() - const checkpoint = await Checkpoint.random(block.checkpointNumber, { numBlocks: 1 }); - checkpoint.blocks = [block]; - return new PublishedCheckpoint( - checkpoint, - new L1PublishedData(BigInt(block.number), BigInt(block.number), Buffer32.random().toString()), - [], - ); - }), + const checkpoints = this.checkpointList.slice(from - 1, from - 1 + limit); + return Promise.resolve( + checkpoints.map(checkpoint => new PublishedCheckpoint(checkpoint, this.mockL1DataForCheckpoint(checkpoint), [])), ); } - public async getCheckpointByArchive(archive: Fr): Promise { - // TODO(mbps): Implement this properly. This only works when we have one block per checkpoint. - const block = this.l2Blocks.find(b => b.archive.root.equals(archive)); - if (!block) { - return undefined; - } - // Create a checkpoint from the block - manually construct since L2Block doesn't have toCheckpoint() - const checkpoint = await Checkpoint.random(block.checkpointNumber, { numBlocks: 1 }); - checkpoint.blocks = [block]; - return checkpoint; + public getCheckpointByArchive(archive: Fr): Promise { + const checkpoint = this.checkpointList.find(c => c.archive.root.equals(archive)); + return Promise.resolve(checkpoint); } public async getCheckpointedBlockByHash(blockHash: BlockHash): Promise { for (const block of this.l2Blocks) { const hash = await block.hash(); if (hash.equals(blockHash)) { - return CheckpointedL2Block.fromFields({ - checkpointNumber: CheckpointNumber.fromBlockNumber(block.number), - block, - l1: new L1PublishedData(BigInt(block.number), BigInt(block.number), Buffer32.random().toString()), - attestations: [], - }); + return this.toCheckpointedBlock(block); } } return undefined; @@ -216,14 +243,7 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { if (!block) { return Promise.resolve(undefined); } - return Promise.resolve( - CheckpointedL2Block.fromFields({ - checkpointNumber: CheckpointNumber.fromBlockNumber(block.number), - block, - l1: new L1PublishedData(BigInt(block.number), BigInt(block.number), Buffer32.random().toString()), - attestations: [], - }), - ); + return Promise.resolve(this.toCheckpointedBlock(block)); } public async getL2BlockByHash(blockHash: BlockHash): Promise { @@ -289,42 +309,36 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { } getCheckpointsForEpoch(epochNumber: EpochNumber): Promise { - // TODO(mbps): Implement this properly. This only works when we have one block per checkpoint. - const epochDuration = DefaultL1ContractsConfig.aztecEpochDuration; - const [start, end] = getSlotRangeForEpoch(epochNumber, { epochDuration }); - const blocks = this.l2Blocks.filter(b => { - const slot = b.header.globalVariables.slotNumber; - return slot >= start && slot <= end; - }); - // Create checkpoints from blocks - manually construct since L2Block doesn't have toCheckpoint() - return Promise.all( - blocks.map(async block => { - const checkpoint = await Checkpoint.random(block.checkpointNumber, { numBlocks: 1 }); - checkpoint.blocks = [block]; - return checkpoint; - }), - ); + return Promise.resolve(this.getCheckpointsInEpoch(epochNumber)); } - getCheckpointedBlocksForEpoch(epochNumber: EpochNumber): Promise { - const epochDuration = DefaultL1ContractsConfig.aztecEpochDuration; - const [start, end] = getSlotRangeForEpoch(epochNumber, { epochDuration }); - const blocks = this.l2Blocks.filter(b => { - const slot = b.header.globalVariables.slotNumber; - return slot >= start && slot <= end; - }); + getCheckpointsDataForEpoch(epochNumber: EpochNumber): Promise { + const checkpoints = this.getCheckpointsInEpoch(epochNumber); return Promise.resolve( - blocks.map(block => - CheckpointedL2Block.fromFields({ - checkpointNumber: CheckpointNumber.fromBlockNumber(block.number), - block, - l1: new L1PublishedData(BigInt(block.number), BigInt(block.number), Buffer32.random().toString()), + checkpoints.map( + (checkpoint): CheckpointData => ({ + checkpointNumber: checkpoint.number, + header: checkpoint.header, + archive: checkpoint.archive, + checkpointOutHash: computeCheckpointOutHash( + checkpoint.blocks.map(b => b.body.txEffects.map(tx => tx.l2ToL1Msgs)), + ), + startBlock: checkpoint.blocks[0].number, + blockCount: checkpoint.blocks.length, attestations: [], + l1: this.mockL1DataForCheckpoint(checkpoint), }), ), ); } + getCheckpointedBlocksForEpoch(epochNumber: EpochNumber): Promise { + const checkpoints = this.getCheckpointsInEpoch(epochNumber); + return Promise.resolve( + checkpoints.flatMap(checkpoint => checkpoint.blocks.map(block => this.toCheckpointedBlock(block))), + ); + } + getBlocksForSlot(slotNumber: SlotNumber): Promise { const blocks = this.l2Blocks.filter(b => b.header.globalVariables.slotNumber === slotNumber); return Promise.resolve(blocks); @@ -413,7 +427,10 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { const makeTipId = (blockId: typeof latestBlockId) => ({ block: blockId, - checkpoint: { number: CheckpointNumber.fromBlockNumber(blockId.number), hash: blockId.hash }, + checkpoint: { + number: this.findCheckpointNumberForBlock(blockId.number) ?? CheckpointNumber(0), + hash: blockId.hash, + }, }); return { @@ -501,4 +518,38 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { getPendingChainValidationStatus(): Promise { return Promise.resolve({ valid: true }); } + + /** Returns checkpoints whose slot falls within the given epoch. */ + private getCheckpointsInEpoch(epochNumber: EpochNumber): Checkpoint[] { + const epochDuration = DefaultL1ContractsConfig.aztecEpochDuration; + const [start, end] = getSlotRangeForEpoch(epochNumber, { epochDuration }); + return this.checkpointList.filter(c => c.header.slotNumber >= start && c.header.slotNumber <= end); + } + + /** Creates a mock L1PublishedData for a checkpoint. */ + private mockL1DataForCheckpoint(checkpoint: Checkpoint): L1PublishedData { + return new L1PublishedData(BigInt(checkpoint.number), BigInt(checkpoint.number), Buffer32.random().toString()); + } + + /** Creates a CheckpointedL2Block from a block using stored checkpoint info. */ + private toCheckpointedBlock(block: L2Block): CheckpointedL2Block { + const checkpoint = this.checkpointList.find(c => c.blocks.some(b => b.number === block.number)); + const checkpointNumber = checkpoint?.number ?? block.checkpointNumber; + return new CheckpointedL2Block( + checkpointNumber, + block, + new L1PublishedData( + BigInt(block.number), + BigInt(block.number), + `0x${block.number.toString(16).padStart(64, '0')}`, + ), + [], + ); + } + + /** Finds the checkpoint number for a block, or undefined if the block is not in any checkpoint. */ + private findCheckpointNumberForBlock(blockNumber: BlockNumber): CheckpointNumber | undefined { + const checkpoint = this.checkpointList.find(c => c.blocks.some(b => b.number === blockNumber)); + return checkpoint?.number; + } } diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts index c228d7694280..2b4efa87d32d 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts @@ -56,13 +56,11 @@ describe('e2e_epochs/epochs_proof_fails', () => { // Here we cause a re-org by not publishing the proof for epoch 0 until after the end of epoch 1 // The proof will be rejected and a re-org will take place - // Ensure that there was at least one block mined in epoch 0, otherwise this test fails, since it + // Ensure that there was at least one checkpoint mined in epoch 0, otherwise this test fails, since it // relies on the proof for epoch zero not landing in time, which will never happen if there is - // nothing to prove on epoch zero. This is flakey because startup times change continuously. - // Also note that there should always be at least a checkpoint before we start since setup - // enforces it (search the comment "waiting for an empty block 1 to be mined" in `setup`). - const firstCheckpointNumber = (await test.monitor.run()).checkpointNumber; - expect(firstCheckpointNumber).toBeGreaterThanOrEqual(CheckpointNumber(1)); + // nothing to prove on epoch zero. We need to wait for the checkpoint L1 tx to be mined, not just + // for the block to appear in the node's world state, since the propose tx may still be in-flight. + await test.waitUntilCheckpointNumber(CheckpointNumber(1)); const firstCheckpoint = await rollup.getCheckpoint(CheckpointNumber(1)); const firstCheckpointEpoch = getEpochAtSlot(firstCheckpoint.slotNumber, test.constants); expect(firstCheckpointEpoch).toEqual(EpochNumber(0)); diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts index f71618d90b37..c3523b9ce36c 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts @@ -24,7 +24,7 @@ import { type P2P, P2PClientState } from '@aztec/p2p'; import type { SlasherClientInterface } from '@aztec/slasher'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { CommitteeAttestation, L2Block, type L2BlockSink, type L2BlockSource } from '@aztec/stdlib/block'; -import { Checkpoint } from '@aztec/stdlib/checkpoint'; +import { Checkpoint, type CheckpointData, L1PublishedData } from '@aztec/stdlib/checkpoint'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { GasFees } from '@aztec/stdlib/gas'; import { @@ -216,7 +216,7 @@ describe('CheckpointProposalJob', () => { l1ToL2MessageSource.getL1ToL2Messages.mockResolvedValue(Array(4).fill(Fr.ZERO)); l2BlockSource = mock(); - l2BlockSource.getCheckpointsForEpoch.mockResolvedValue([]); + l2BlockSource.getCheckpointsDataForEpoch.mockResolvedValue([]); blockSink = mock(); blockSink.addBlock.mockResolvedValue(undefined); @@ -369,6 +369,7 @@ describe('CheckpointProposalJob', () => { it('passes previous checkpoint out hashes when there are earlier checkpoints in the epoch', async () => { // Create two previous checkpoints in the same epoch const previousCheckpoints = await timesAsync(2, i => Checkpoint.random(CheckpointNumber(i + 1))); + const previousCheckpointsData: CheckpointData[] = previousCheckpoints.map(c => toCheckpointData(c)); // Update job to be for checkpoint 3 checkpointNumber = CheckpointNumber(3); @@ -383,7 +384,7 @@ describe('CheckpointProposalJob', () => { ); // Mock l2BlockSource to return the previous checkpoints - l2BlockSource.getCheckpointsForEpoch.mockResolvedValue(previousCheckpoints); + l2BlockSource.getCheckpointsDataForEpoch.mockResolvedValue(previousCheckpointsData); // Build block successfully const { txs, block } = await setupTxsAndBlock(p2p, globalVariables, 1, chainId); @@ -419,8 +420,12 @@ describe('CheckpointProposalJob', () => { }), ); - // Mock l2BlockSource to return all three checkpoints - l2BlockSource.getCheckpointsForEpoch.mockResolvedValue([previousCheckpoint, currentCheckpoint, futureCheckpoint]); + // Mock l2BlockSource to return all three checkpoints as data + l2BlockSource.getCheckpointsDataForEpoch.mockResolvedValue([ + toCheckpointData(previousCheckpoint), + toCheckpointData(currentCheckpoint), + toCheckpointData(futureCheckpoint), + ]); // Build block successfully const { txs, block } = await setupTxsAndBlock(p2p, globalVariables, 1, chainId); @@ -1114,3 +1119,17 @@ class TestCheckpointProposalJob extends CheckpointProposalJob { return super.buildSingleBlock(checkpointBuilder, opts); } } + +/** Creates a CheckpointData from a Checkpoint for testing. */ +function toCheckpointData(checkpoint: Checkpoint): CheckpointData { + return { + checkpointNumber: checkpoint.number, + header: checkpoint.header, + archive: checkpoint.archive, + checkpointOutHash: checkpoint.getCheckpointOutHash(), + startBlock: BlockNumber(checkpoint.blocks[0]?.number ?? 1), + blockCount: checkpoint.blocks.length, + attestations: [], + l1: L1PublishedData.random(), + }; +} diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts index 00f39be89c90..6b373ad5a3dd 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts @@ -422,7 +422,7 @@ describe('CheckpointProposalJob Timing Tests', () => { l1ToL2MessageSource.getL1ToL2Messages.mockResolvedValue(Array(4).fill(Fr.ZERO)); l2BlockSource = mock(); - l2BlockSource.getCheckpointsForEpoch.mockResolvedValue([]); + l2BlockSource.getCheckpointsDataForEpoch.mockResolvedValue([]); blockSink = mock(); blockSink.addBlock.mockResolvedValue(undefined); diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index fcc773ab3a5d..032804d9cd04 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -186,10 +186,9 @@ export class CheckpointProposalJob implements Traceable { const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages); // Collect the out hashes of all the checkpoints before this one in the same epoch - const previousCheckpoints = (await this.l2BlockSource.getCheckpointsForEpoch(this.epoch)).filter( - c => c.number < this.checkpointNumber, - ); - const previousCheckpointOutHashes = previousCheckpoints.map(c => c.getCheckpointOutHash()); + const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.epoch)) + .filter(c => c.checkpointNumber < this.checkpointNumber) + .map(c => c.checkpointOutHash); // Get the fee asset price modifier from the oracle const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier(); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index 9329b90fba22..cb625f07002d 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -271,6 +271,7 @@ describe('sequencer', () => { getPendingChainValidationStatus: mockFn().mockResolvedValue({ valid: true }), getCheckpointedBlocksForEpoch: mockFn().mockResolvedValue([]), getCheckpointsForEpoch: mockFn().mockResolvedValue([]), + getCheckpointsDataForEpoch: mockFn().mockResolvedValue([]), }); l1ToL2MessageSource = mock({ diff --git a/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts b/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts index 24355f8545d8..dd52040bdef7 100644 --- a/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts +++ b/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts @@ -253,6 +253,7 @@ describe('EpochPruneWatcher', () => { class MockL2BlockSource { public readonly events = new EventEmitter(); public getCheckpointsForEpoch = () => []; + public getCheckpointsDataForEpoch = () => []; constructor() {} } diff --git a/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts b/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts index f10c4b39378c..0de0f6b27f65 100644 --- a/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts +++ b/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts @@ -132,9 +132,9 @@ export class EpochPruneWatcher extends (EventEmitter as new () => WatcherEmitter const blocksByCheckpoint = chunkBy(sortedBlocks, b => b.checkpointNumber); // Get prior checkpoints in the epoch (in case this was a partial prune) to extract the out hashes - const priorCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsForEpoch(epochNumber)) - .filter(c => c.number < sortedBlocks[0].checkpointNumber) - .map(c => c.getCheckpointOutHash()); + const priorCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(epochNumber)) + .filter(c => c.checkpointNumber < sortedBlocks[0].checkpointNumber) + .map(c => c.checkpointOutHash); let previousCheckpointOutHashes: Fr[] = [...priorCheckpointOutHashes]; const fork = await this.checkpointsBuilder.getFork( diff --git a/yarn-project/stdlib/src/block/l2_block_source.ts b/yarn-project/stdlib/src/block/l2_block_source.ts index 778c70c97228..f1daf550d7eb 100644 --- a/yarn-project/stdlib/src/block/l2_block_source.ts +++ b/yarn-project/stdlib/src/block/l2_block_source.ts @@ -13,6 +13,7 @@ import type { TypedEventEmitter } from '@aztec/foundation/types'; import { z } from 'zod'; import type { Checkpoint } from '../checkpoint/checkpoint.js'; +import type { CheckpointData } from '../checkpoint/checkpoint_data.js'; import type { PublishedCheckpoint } from '../checkpoint/published_checkpoint.js'; import type { L1RollupConstants } from '../epoch-helpers/index.js'; import { CheckpointHeader } from '../rollup/checkpoint_header.js'; @@ -99,6 +100,12 @@ export interface L2BlockSource { */ getCheckpointsForEpoch(epochNumber: EpochNumber): Promise; + /** + * Gets lightweight checkpoint metadata for a given epoch, without fetching full block data. + * @param epochNumber - Epoch for which we want checkpoint data + */ + getCheckpointsDataForEpoch(epochNumber: EpochNumber): Promise; + /** * Gets a block header by its hash. * @param blockHash - The block hash to retrieve. diff --git a/yarn-project/stdlib/src/checkpoint/checkpoint.ts b/yarn-project/stdlib/src/checkpoint/checkpoint.ts index 9e633345f89c..2c95d3c0be4a 100644 --- a/yarn-project/stdlib/src/checkpoint/checkpoint.ts +++ b/yarn-project/stdlib/src/checkpoint/checkpoint.ts @@ -94,9 +94,11 @@ export class Checkpoint { return this.header.hash(); } - // Returns the out hash computed from all l2-to-l1 messages in this checkpoint. - // Note: This value is different from the out hash in the header, which is the **accumulated** out hash over all - // checkpoints up to and including this one in the epoch. + /** + * Returns the out hash computed from all l2-to-l1 messages in this checkpoint. + * Note: This value is different from the out hash in the header, which is the **accumulated** out hash over all + * checkpoints up to and including this one in the epoch. + */ public getCheckpointOutHash(): Fr { const msgs = this.blocks.map(block => block.body.txEffects.map(txEffect => txEffect.l2ToL1Msgs)); return computeCheckpointOutHash(msgs); diff --git a/yarn-project/stdlib/src/checkpoint/checkpoint_data.ts b/yarn-project/stdlib/src/checkpoint/checkpoint_data.ts new file mode 100644 index 000000000000..32dd799181ee --- /dev/null +++ b/yarn-project/stdlib/src/checkpoint/checkpoint_data.ts @@ -0,0 +1,51 @@ +import { + BlockNumber, + BlockNumberSchema, + CheckpointNumber, + CheckpointNumberSchema, +} from '@aztec/foundation/branded-types'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { schemas } from '@aztec/foundation/schemas'; + +import { z } from 'zod'; + +import { CommitteeAttestation } from '../block/proposal/committee_attestation.js'; +import { CheckpointHeader } from '../rollup/checkpoint_header.js'; +import { AppendOnlyTreeSnapshot } from '../trees/append_only_tree_snapshot.js'; +import { L1PublishedData } from './published_checkpoint.js'; + +/** Lightweight checkpoint metadata without full block data. */ +export type CheckpointData = { + checkpointNumber: CheckpointNumber; + header: CheckpointHeader; + archive: AppendOnlyTreeSnapshot; + checkpointOutHash: Fr; + startBlock: BlockNumber; + blockCount: number; + attestations: CommitteeAttestation[]; + l1: L1PublishedData; +}; + +export const CheckpointDataSchema = z + .object({ + checkpointNumber: CheckpointNumberSchema, + header: CheckpointHeader.schema, + archive: AppendOnlyTreeSnapshot.schema, + checkpointOutHash: schemas.Fr, + startBlock: BlockNumberSchema, + blockCount: schemas.Integer, + attestations: z.array(CommitteeAttestation.schema), + l1: L1PublishedData.schema, + }) + .transform( + (obj): CheckpointData => ({ + checkpointNumber: obj.checkpointNumber, + header: obj.header, + archive: obj.archive, + checkpointOutHash: obj.checkpointOutHash, + startBlock: obj.startBlock, + blockCount: obj.blockCount, + attestations: obj.attestations, + l1: obj.l1, + }), + ); diff --git a/yarn-project/stdlib/src/checkpoint/index.ts b/yarn-project/stdlib/src/checkpoint/index.ts index 6c189e5a5ddc..d86f88c87bbb 100644 --- a/yarn-project/stdlib/src/checkpoint/index.ts +++ b/yarn-project/stdlib/src/checkpoint/index.ts @@ -1,3 +1,4 @@ export * from './checkpoint.js'; +export * from './checkpoint_data.js'; export * from './checkpoint_info.js'; export * from './published_checkpoint.js'; diff --git a/yarn-project/stdlib/src/checkpoint/published_checkpoint.ts b/yarn-project/stdlib/src/checkpoint/published_checkpoint.ts index d5afc5c2e3e0..67c2104fe05b 100644 --- a/yarn-project/stdlib/src/checkpoint/published_checkpoint.ts +++ b/yarn-project/stdlib/src/checkpoint/published_checkpoint.ts @@ -55,9 +55,11 @@ export class L1PublishedData { export class PublishedCheckpoint { constructor( + /** The checkpoint itself. */ public checkpoint: Checkpoint, + /** Info on when this checkpoint was published on L1. */ public l1: L1PublishedData, - // The attestations for the last block in the checkpoint. + /** The attestations for the last block in the checkpoint. */ public attestations: CommitteeAttestation[], ) {} diff --git a/yarn-project/stdlib/src/interfaces/archiver.test.ts b/yarn-project/stdlib/src/interfaces/archiver.test.ts index 6235da1d6c81..2b5cb983325b 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.test.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.test.ts @@ -14,6 +14,7 @@ import { type BlockData, BlockHash, CommitteeAttestation, L2Block } from '../blo import type { L2Tips } from '../block/l2_block_source.js'; import type { ValidateCheckpointResult } from '../block/validate_block_result.js'; import { Checkpoint } from '../checkpoint/checkpoint.js'; +import type { CheckpointData } from '../checkpoint/checkpoint_data.js'; import { L1PublishedData, PublishedCheckpoint } from '../checkpoint/published_checkpoint.js'; import { getContractClassFromArtifact } from '../contract/contract_class.js'; import { @@ -194,6 +195,14 @@ describe('ArchiverApiSchema', () => { expect(result).toEqual([expect.any(Checkpoint)]); }); + it('getCheckpointsDataForEpoch', async () => { + const result = await context.client.getCheckpointsDataForEpoch(EpochNumber(1)); + expect(result).toHaveLength(1); + expect(result[0].checkpointNumber).toBeDefined(); + expect(result[0].checkpointOutHash).toBeDefined(); + expect(result[0].attestations[0]).toBeInstanceOf(CommitteeAttestation); + }); + it('getCheckpointedBlock', async () => { const result = await context.client.getCheckpointedBlock(BlockNumber(1)); expect(result).toBeDefined(); @@ -501,6 +510,22 @@ class MockArchiver implements ArchiverApi { expect(epochNumber).toEqual(EpochNumber(1)); return [await Checkpoint.random(CheckpointNumber(1))]; } + async getCheckpointsDataForEpoch(epochNumber: EpochNumber): Promise { + expect(epochNumber).toEqual(EpochNumber(1)); + const checkpoint = await Checkpoint.random(CheckpointNumber(1)); + return [ + { + checkpointNumber: checkpoint.number, + header: checkpoint.header, + archive: checkpoint.archive, + checkpointOutHash: checkpoint.getCheckpointOutHash(), + startBlock: BlockNumber(1), + blockCount: checkpoint.blocks.length, + attestations: [CommitteeAttestation.random()], + l1: L1PublishedData.random(), + }, + ]; + } async getCheckpointedBlocksForEpoch(epochNumber: EpochNumber): Promise { expect(epochNumber).toEqual(EpochNumber(1)); const block = await L2Block.random(BlockNumber(Number(epochNumber))); diff --git a/yarn-project/stdlib/src/interfaces/archiver.ts b/yarn-project/stdlib/src/interfaces/archiver.ts index 4caf08b4a30a..9af2b49e6fbc 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.ts @@ -11,6 +11,7 @@ import { L2Block } from '../block/l2_block.js'; import { type L2BlockSource, L2TipsSchema } from '../block/l2_block_source.js'; import { ValidateCheckpointResultSchema } from '../block/validate_block_result.js'; import { Checkpoint } from '../checkpoint/checkpoint.js'; +import { CheckpointDataSchema } from '../checkpoint/checkpoint_data.js'; import { PublishedCheckpoint } from '../checkpoint/published_checkpoint.js'; import { ContractClassPublicSchema, @@ -115,6 +116,7 @@ export const ArchiverApiSchema: ApiSchemaFor = { getL2SlotNumber: z.function().args().returns(schemas.SlotNumber.optional()), getL2EpochNumber: z.function().args().returns(EpochNumberSchema.optional()), getCheckpointsForEpoch: z.function().args(EpochNumberSchema).returns(z.array(Checkpoint.schema)), + getCheckpointsDataForEpoch: z.function().args(EpochNumberSchema).returns(z.array(CheckpointDataSchema)), getCheckpointedBlocksForEpoch: z.function().args(EpochNumberSchema).returns(z.array(CheckpointedL2Block.schema)), getBlocksForSlot: z.function().args(schemas.SlotNumber).returns(z.array(L2Block.schema)), getCheckpointedBlockHeadersForEpoch: z.function().args(EpochNumberSchema).returns(z.array(BlockHeader.schema)), diff --git a/yarn-project/validator-client/src/block_proposal_handler.ts b/yarn-project/validator-client/src/block_proposal_handler.ts index 4be070f3a8c5..776b48ac266c 100644 --- a/yarn-project/validator-client/src/block_proposal_handler.ts +++ b/yarn-project/validator-client/src/block_proposal_handler.ts @@ -1,7 +1,6 @@ import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; import type { EpochCache } from '@aztec/epoch-cache'; import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { chunkBy } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { TimeoutError } from '@aztec/foundation/error'; import { createLogger } from '@aztec/foundation/log'; @@ -12,11 +11,7 @@ import { BlockProposalValidator } from '@aztec/p2p/msg_validators'; import type { BlockData, L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; import { getEpochAtSlot, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import type { ITxProvider, ValidatorClientFullConfig, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; -import { - type L1ToL2MessageSource, - computeCheckpointOutHash, - computeInHashFromL1ToL2Messages, -} from '@aztec/stdlib/messaging'; +import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging'; import type { BlockProposal } from '@aztec/stdlib/p2p'; import type { CheckpointGlobalVariables, FailedTx, Tx } from '@aztec/stdlib/tx'; import { @@ -218,17 +213,11 @@ export class BlockProposalHandler { // Try re-executing the transactions in the proposal if needed let reexecutionResult; if (shouldReexecute) { - // Compute the previous checkpoint out hashes for the epoch. - // TODO(leila/mbps): There can be a more efficient way to get the previous checkpoint out - // hashes without having to fetch all the blocks. + // Collect the out hashes of all the checkpoints before this one in the same epoch const epoch = getEpochAtSlot(slotNumber, this.epochCache.getL1Constants()); - const checkpointedBlocks = (await this.blockSource.getCheckpointedBlocksForEpoch(epoch)) - .filter(b => b.block.number < blockNumber) - .sort((a, b) => a.block.number - b.block.number); - const blocksByCheckpoint = chunkBy(checkpointedBlocks, b => b.checkpointNumber); - const previousCheckpointOutHashes = blocksByCheckpoint.map(checkpointBlocks => - computeCheckpointOutHash(checkpointBlocks.map(b => b.block.body.txEffects.map(tx => tx.l2ToL1Msgs))), - ); + const previousCheckpointOutHashes = (await this.blockSource.getCheckpointsDataForEpoch(epoch)) + .filter(c => c.checkpointNumber < checkpointNumber) + .map(c => c.checkpointOutHash); try { this.log.verbose(`Re-executing transactions in the proposal`, proposalInfo); diff --git a/yarn-project/validator-client/src/validator.test.ts b/yarn-project/validator-client/src/validator.test.ts index feb64eb9aba9..df6d16208a16 100644 --- a/yarn-project/validator-client/src/validator.test.ts +++ b/yarn-project/validator-client/src/validator.test.ts @@ -96,6 +96,7 @@ describe('ValidatorClient', () => { >[1] as any); blockSource = mock(); blockSource.getCheckpointedBlocksForEpoch.mockResolvedValue([]); + blockSource.getCheckpointsDataForEpoch.mockResolvedValue([]); blockSource.getBlocksForSlot.mockResolvedValue([]); epochCache.isEscapeHatchOpenAtSlot.mockResolvedValue(false); l1ToL2MessageSource = mock(); diff --git a/yarn-project/validator-client/src/validator.ts b/yarn-project/validator-client/src/validator.ts index 873dc9a15b50..29c5d497576e 100644 --- a/yarn-project/validator-client/src/validator.ts +++ b/yarn-project/validator-client/src/validator.ts @@ -662,14 +662,11 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) // Get L1-to-L2 messages for this checkpoint const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpointNumber); - // Compute the previous checkpoint out hashes for the epoch. - // TODO: There can be a more efficient way to get the previous checkpoint out hashes without having to fetch the - // actual checkpoints and the blocks/txs in them. + // Collect the out hashes of all the checkpoints before this one in the same epoch const epoch = getEpochAtSlot(slot, this.epochCache.getL1Constants()); - const previousCheckpoints = (await this.blockSource.getCheckpointsForEpoch(epoch)) - .filter(b => b.number < checkpointNumber) - .sort((a, b) => a.number - b.number); - const previousCheckpointOutHashes = previousCheckpoints.map(c => c.getCheckpointOutHash()); + const previousCheckpointOutHashes = (await this.blockSource.getCheckpointsDataForEpoch(epoch)) + .filter(c => c.checkpointNumber < checkpointNumber) + .map(c => c.checkpointOutHash); // Fork world state at the block before the first block const parentBlockNumber = BlockNumber(firstBlock.number - 1);