From 023dcedd7a1cf06e4fd4465bcf9c9729cb021c24 Mon Sep 17 00:00:00 2001 From: benesjan Date: Thu, 7 Mar 2024 10:28:15 +0000 Subject: [PATCH 01/14] feat: updating archiver with new inbox --- .../archiver/src/archiver/archiver.ts | 10 ++++ .../archiver/src/archiver/archiver_store.ts | 10 ++++ .../archiver/src/archiver/data_retrieval.ts | 43 ++++++++++++++- .../archiver/src/archiver/eth_log_handlers.ts | 52 ++++++++++++++++++- .../kv_archiver_store/kv_archiver_store.ts | 10 ++++ .../kv_archiver_store/message_store.ts | 39 ++++++++++++++ .../circuit-types/src/l1_to_l2_message.ts | 31 +++++++++++ .../scripts/generate-artifacts.sh | 1 + 8 files changed, 194 insertions(+), 2 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 01063092229f..81f97bdad24a 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -43,6 +43,7 @@ import { retrieveBlockMetadataFromRollup, retrieveNewCancelledL1ToL2Messages, retrieveNewContractData, + retrieveNewL1ToL2Messages, retrieveNewPendingL1ToL2Messages, } from './data_retrieval.js'; @@ -192,6 +193,15 @@ export class Archiver implements ArchiveSource { // ********** Events that are processed per L1 block ********** + const retrievedNewL1ToL2Messages = await retrieveNewL1ToL2Messages( + this.publicClient, + this.inboxAddress, + blockUntilSynced, + lastL1Blocks.addedMessages + 1n, + currentL1BlockNumber, + ); + + // TODO(#4492): Nuke the following when purging the old inbox // Process l1ToL2Messages, these are consumed as time passes, not each block const retrievedPendingL1ToL2Messages = await retrieveNewPendingL1ToL2Messages( this.publicClient, diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 134c224cf005..3a17c48057aa 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -90,11 +90,20 @@ export interface ArchiverDataStore { blockNumber: number, ): Promise; + /** + * Append new L1 to L2 messages to the store. + * @param messages - The L1 to L2 messages to be added to the store. + * @param l1BlockNumber - The block number of the L1 block that added the messages. + * @returns True if the operation is successful. + */ + addNewL1ToL2Messages(messages: L1ToL2Message[], l1BlockNumber: bigint): Promise; + /** * Append new pending L1 to L2 messages to the store. * @param messages - The L1 to L2 messages to be added to the store. * @param l1BlockNumber - The block number of the L1 block that added the messages. * @returns True if the operation is successful. + * TODO(#4492): Nuke the following when purging the old inbox */ addPendingL1ToL2Messages(messages: L1ToL2Message[], l1BlockNumber: bigint): Promise; @@ -103,6 +112,7 @@ export interface ArchiverDataStore { * @param entryKeys - The entry keys to be removed from the store. * @param l1BlockNumber - The block number of the L1 block that cancelled the messages. * @returns True if the operation is successful. + * TODO(#4492): Nuke the following when purging the old inbox */ cancelPendingL1ToL2EntryKeys(entryKeys: Fr[], l1BlockNumber: bigint): Promise; diff --git a/yarn-project/archiver/src/archiver/data_retrieval.ts b/yarn-project/archiver/src/archiver/data_retrieval.ts index 9ac40aaac344..4ba5cf90ece6 100644 --- a/yarn-project/archiver/src/archiver/data_retrieval.ts +++ b/yarn-project/archiver/src/archiver/data_retrieval.ts @@ -1,4 +1,4 @@ -import { Body, ExtendedContractData, L1ToL2Message } from '@aztec/circuit-types'; +import { Body, ExtendedContractData, L1ToL2Message, NewInboxLeaf } from '@aztec/circuit-types'; import { AppendOnlyTreeSnapshot, Fr, Header } from '@aztec/circuits.js'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -8,11 +8,13 @@ import { getContractDeploymentLogs, getL1ToL2MessageCancelledLogs, getL2BlockProcessedLogs, + getLeafInsertedLogs, getPendingL1ToL2MessageLogs, getTxsPublishedLogs, processCancelledL1ToL2MessagesLogs, processContractDeploymentLogs, processL2BlockProcessedLogs, + processNewLeafInsertedLogs, processPendingL1ToL2MessageAddedLogs, processTxsPublishedLogs, } from './eth_log_handlers.js'; @@ -192,6 +194,44 @@ export async function retrieveNewPendingL1ToL2Messages( return { nextEthBlockNumber: searchStartBlock, retrievedData: retrievedNewL1ToL2Messages }; } +/** + * Fetch new L1 to L2 messages. + * @param publicClient - The viem public client to use for transaction retrieval. + * @param newInboxAddress - The address of the inbox contract to fetch messages from. + * @param blockUntilSynced - If true, blocks until the archiver has fully synced. + * @param searchStartBlock - The block number to use for starting the search. + * @param searchEndBlock - The highest block number that we should search up to. + * @returns An array of NewInboxLeaf and next eth block to search from. + */ +export async function retrieveNewL1ToL2Messages( + publicClient: PublicClient, + newInboxAddress: EthAddress, + blockUntilSynced: boolean, + searchStartBlock: bigint, + searchEndBlock: bigint, +): Promise> { + const retrievedNewL1ToL2Messages: [NewInboxLeaf, bigint][] = []; + do { + if (searchStartBlock > searchEndBlock) { + break; + } + const leafInsertedLogs = await getLeafInsertedLogs( + publicClient, + newInboxAddress, + searchStartBlock, + searchEndBlock, + ); + if (leafInsertedLogs.length === 0) { + break; + } + const newL1ToL2Messages = processNewLeafInsertedLogs(leafInsertedLogs); + retrievedNewL1ToL2Messages.push(...newL1ToL2Messages); + // handles the case when there are no new messages: + searchStartBlock = (leafInsertedLogs.findLast(msgLog => !!msgLog)?.blockNumber || searchStartBlock) + 1n; + } while (blockUntilSynced && searchStartBlock <= searchEndBlock); + return { nextEthBlockNumber: searchStartBlock, retrievedData: retrievedNewL1ToL2Messages }; +} + /** * Fetch newly cancelled L1 to L2 messages. * @param publicClient - The viem public client to use for transaction retrieval. @@ -200,6 +240,7 @@ export async function retrieveNewPendingL1ToL2Messages( * @param searchStartBlock - The block number to use for starting the search. * @param searchEndBlock - The highest block number that we should search up to. * @returns An array of entry keys that were cancelled and next eth block to search from. + * TODO(#4492): Nuke the following when purging the old inbox */ export async function retrieveNewCancelledL1ToL2Messages( publicClient: PublicClient, diff --git a/yarn-project/archiver/src/archiver/eth_log_handlers.ts b/yarn-project/archiver/src/archiver/eth_log_handlers.ts index b24445a35e2b..19e1009a5e6f 100644 --- a/yarn-project/archiver/src/archiver/eth_log_handlers.ts +++ b/yarn-project/archiver/src/archiver/eth_log_handlers.ts @@ -6,16 +6,41 @@ import { L1Actor, L1ToL2Message, L2Actor, + NewInboxLeaf, } from '@aztec/circuit-types'; import { AppendOnlyTreeSnapshot, Header } from '@aztec/circuits.js'; import { AztecAddress } from '@aztec/foundation/aztec-address'; import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr } from '@aztec/foundation/fields'; import { BufferReader, numToUInt32BE } from '@aztec/foundation/serialize'; -import { AvailabilityOracleAbi, ContractDeploymentEmitterAbi, InboxAbi, RollupAbi } from '@aztec/l1-artifacts'; +import { AvailabilityOracleAbi, ContractDeploymentEmitterAbi, InboxAbi, NewInboxAbi, RollupAbi } from '@aztec/l1-artifacts'; import { Hex, Log, PublicClient, decodeFunctionData, getAbiItem, getAddress, hexToBytes } from 'viem'; +/** + * Processes newly received LeafInserted (L1 to L2) logs. + * @param logs - LeafInserted logs. + * @returns Array of all Pending L1 to L2 messages that were processed + */ +export function processNewLeafInsertedLogs( + logs: Log[], +): [NewInboxLeaf, bigint][] { + const leaves: [NewInboxLeaf, bigint][] = []; + for (const log of logs) { + const { blockNumber, index, value } = + log.args; + leaves.push([ + new NewInboxLeaf( + blockNumber, + index, + Buffer.from(hexToBytes(value)), + ), + log.blockNumber!, + ]); + } + return leaves; +} + /** * Processes newly received MessageAdded (L1 to L2) logs. * @param logs - MessageAdded logs. @@ -340,3 +365,28 @@ export function getL1ToL2MessageCancelledLogs( toBlock: toBlock + 1n, // the toBlock argument in getLogs is exclusive }); } + +/** + * Get relevant `LeafInserted` logs emitted by NewInbox on chain. + * @param publicClient - The viem public client to use for transaction retrieval. + * @param newInboxAddress - The address of the new inbox contract. + * @param fromBlock - First block to get logs from (inclusive). + * @param toBlock - Last block to get logs from (inclusive). + * @returns An array of `LeafInserted` logs. + */ +export function getLeafInsertedLogs( + publicClient: PublicClient, + newInboxAddress: EthAddress, + fromBlock: bigint, + toBlock: bigint, +): Promise[]> { + return publicClient.getLogs({ + address: getAddress(newInboxAddress.toString()), + event: getAbiItem({ + abi: NewInboxAbi, + name: 'LeafInserted', + }), + fromBlock, + toBlock: toBlock + 1n, // the toBlock argument in getLogs is exclusive + }); +} \ No newline at end of file diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index fba3d04c7da9..874894d52878 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -148,6 +148,16 @@ export class KVArchiverDataStore implements ArchiverDataStore { return this.#logStore.addLogs(encryptedLogs, unencryptedLogs, blockNumber); } + /** + * Append new L1 to L2 messages to the store. + * @param messages - The L1 to L2 messages to be added to the store. + * @param l1BlockNumber - The L1 block number for which to add the messages. + * @returns True if the operation is successful. + */ + addNewL1ToL2Messages(messages: L1ToL2Message[], l1BlockNumber: bigint): Promise { + return Promise.resolve(this.#messageStore.addNewL1ToL2Messages(messages, l1BlockNumber)); + } + /** * Append new pending L1 to L2 messages to the store. * @param messages - The L1 to L2 messages to be added to the store. diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts index 641a8421cb55..243a12abca0d 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts @@ -21,6 +21,8 @@ type Message = { export class MessageStore { #messages: AztecMap; #pendingMessagesByFee: AztecCounter<[number, string]>; + #lastL1BlockNewMessages: AztecSingleton; + // TODO(#4492): Nuke the following when purging the old inbox #lastL1BlockAddingMessages: AztecSingleton; #lastL1BlockCancellingMessages: AztecSingleton; @@ -29,6 +31,7 @@ export class MessageStore { constructor(private db: AztecKVStore) { this.#messages = db.openMap('archiver_l1_to_l2_messages'); this.#pendingMessagesByFee = db.openCounter('archiver_messages_by_fee'); + this.#lastL1BlockNewMessages = db.openSingleton('archiver_last_l1_block_new_messages'); this.#lastL1BlockAddingMessages = db.openSingleton('archiver_last_l1_block_adding_messages'); this.#lastL1BlockCancellingMessages = db.openSingleton('archiver_last_l1_block_cancelling_messages'); } @@ -39,11 +42,47 @@ export class MessageStore { */ getL1BlockNumber() { return { + newMessages: this.#lastL1BlockNewMessages.get() ?? 0n, + // TODO(#4492): Nuke the following when purging the old inbox addedMessages: this.#lastL1BlockAddingMessages.get() ?? 0n, cancelledMessages: this.#lastL1BlockCancellingMessages.get() ?? 0n, }; } + /** + * Append new L1 to L2 messages to the store. + * @param messages - The L1 to L2 messages to be added to the store. + * @param l1BlockNumber - The L1 block number for which to add the messages. + * @returns True if the operation is successful. + */ + addNewL1ToL2Messages(messages: L1ToL2Message[], l1BlockNumber: bigint): Promise { + return this.db.transaction(() => { + const lastL1BlockNumber = this.#lastL1BlockAddingMessages.get() ?? 0n; + if (lastL1BlockNumber >= l1BlockNumber) { + return false; + } + + void this.#lastL1BlockNewMessages.set(l1BlockNumber); + + for (const message of messages) { + const entryKey = message.entryKey?.toString(); + if (!entryKey) { + throw new Error('Message does not have an entry key'); + } + + void this.#messages.setIfNotExists(entryKey, { + message: message.toBuffer(), + fee: message.fee, + confirmed: false, + }); + + void this.#pendingMessagesByFee.update([message.fee, entryKey], 1); + } + + return true; + }); + } + /** * Append new pending L1 to L2 messages to the store. * @param messages - The L1 to L2 messages to be added to the store. diff --git a/yarn-project/circuit-types/src/l1_to_l2_message.ts b/yarn-project/circuit-types/src/l1_to_l2_message.ts index 09bbbad70acc..5830b23cb7cc 100644 --- a/yarn-project/circuit-types/src/l1_to_l2_message.ts +++ b/yarn-project/circuit-types/src/l1_to_l2_message.ts @@ -31,8 +31,36 @@ export interface L1ToL2MessageSource { getBlockNumber(): Promise; } +export class NewInboxLeaf { + constructor( + /** L2 block number in which the message will be included. */ + public readonly blockNumber: bigint, + /** Index of the leaf in L2 block message subtree. */ + public readonly index: bigint, + /** Leaf of the subtree. */ + public readonly leaf: Buffer, + ) { + if (leaf.length !== 32) { + throw new Error('Invalid leaf length'); + } + } + + toBuffer(): Buffer { + return Buffer.concat([toBufferBE(this.blockNumber, 32), toBufferBE(this.index, 32), this.leaf]); + } + + fromBuffer(buffer: Buffer | BufferReader): NewInboxLeaf { + const reader = BufferReader.asReader(buffer); + const blockNumber = toBigIntBE(reader.readBytes(32)); + const index = toBigIntBE(reader.readBytes(32)); + const leaf = reader.readBytes(32); + return new NewInboxLeaf(blockNumber, index, leaf); + } +} + /** * L1AndL2Message and Index (in the merkle tree) as one type + * TODO(#4492): Nuke the following when purging the old inbox */ export class L1ToL2MessageAndIndex { constructor( @@ -65,6 +93,7 @@ export class L1ToL2MessageAndIndex { /** * The format of an L1 to L2 Message. + * TODO(#4492): Nuke the following when purging the old inbox */ export class L1ToL2Message { constructor( @@ -160,6 +189,7 @@ export class L1ToL2Message { /** * The sender of an L1 to L2 message. + * TODO(#4492): Move to separate file when purging the old inbox */ export class L1Actor { constructor( @@ -199,6 +229,7 @@ export class L1Actor { /** * The recipient of an L2 message. + * TODO(#4492): Move to separate file when purging the old inbox */ export class L2Actor { constructor( diff --git a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh index c7f47dd31274..9c39b9526d1b 100755 --- a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh +++ b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh @@ -13,6 +13,7 @@ CONTRACTS=( "l1-contracts:AvailabilityOracle" "l1-contracts:Registry" "l1-contracts:Inbox" + "l1-contracts:NewInbox" "l1-contracts:Outbox" "l1-contracts:Rollup" "l1-contracts:ContractDeploymentEmitter" From 700d1862dc7b70a68265595d70a22fcd2c789928 Mon Sep 17 00:00:00 2001 From: benesjan Date: Thu, 7 Mar 2024 11:06:26 +0000 Subject: [PATCH 02/14] WIP --- .../archiver/src/archiver/archiver.ts | 22 +++++++++----- .../archiver/src/archiver/archiver_store.ts | 5 ++-- .../archiver/src/archiver/data_retrieval.ts | 15 ++++------ .../archiver/src/archiver/eth_log_handlers.ts | 30 +++++++++---------- 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 81f97bdad24a..bc5173193d93 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -193,14 +193,6 @@ export class Archiver implements ArchiveSource { // ********** Events that are processed per L1 block ********** - const retrievedNewL1ToL2Messages = await retrieveNewL1ToL2Messages( - this.publicClient, - this.inboxAddress, - blockUntilSynced, - lastL1Blocks.addedMessages + 1n, - currentL1BlockNumber, - ); - // TODO(#4492): Nuke the following when purging the old inbox // Process l1ToL2Messages, these are consumed as time passes, not each block const retrievedPendingL1ToL2Messages = await retrieveNewPendingL1ToL2Messages( @@ -245,6 +237,20 @@ export class Archiver implements ArchiveSource { // ********** Events that are processed per L2 block ********** + const retrievedNewL1ToL2Messages = await retrieveNewL1ToL2Messages( + this.publicClient, + this.inboxAddress, + blockUntilSynced, + lastL1Blocks.addedMessages + 1n, + currentL1BlockNumber, + ); + await this.store.addNewL1ToL2Messages( + retrievedNewL1ToL2Messages.retrievedData, + // -1n because the function expects the last block in which the message was emitted and not the one after next + // TODO(#4492): Check whether this could be cleaned up - `nextEthBlockNumber` value doesn't seem to be used much + retrievedNewL1ToL2Messages.nextEthBlockNumber - 1n, + ); + // Read all data from chain and then write to our stores at the end const nextExpectedL2BlockNum = BigInt((await this.store.getBlockNumber()) + 1); diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 3a17c48057aa..f6327e053d07 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -8,6 +8,7 @@ import { L2BlockL2Logs, LogFilter, LogType, + NewInboxLeaf, TxEffect, TxHash, TxReceipt, @@ -93,10 +94,10 @@ export interface ArchiverDataStore { /** * Append new L1 to L2 messages to the store. * @param messages - The L1 to L2 messages to be added to the store. - * @param l1BlockNumber - The block number of the L1 block that added the messages. + * @param lastMessageL1BlockNumber - The L1 block number in which the last message was emitted. * @returns True if the operation is successful. */ - addNewL1ToL2Messages(messages: L1ToL2Message[], l1BlockNumber: bigint): Promise; + addNewL1ToL2Messages(messages: NewInboxLeaf[], lastMessageL1BlockNumber: bigint): Promise; /** * Append new pending L1 to L2 messages to the store. diff --git a/yarn-project/archiver/src/archiver/data_retrieval.ts b/yarn-project/archiver/src/archiver/data_retrieval.ts index 4ba5cf90ece6..ec9378c7d029 100644 --- a/yarn-project/archiver/src/archiver/data_retrieval.ts +++ b/yarn-project/archiver/src/archiver/data_retrieval.ts @@ -14,7 +14,7 @@ import { processCancelledL1ToL2MessagesLogs, processContractDeploymentLogs, processL2BlockProcessedLogs, - processNewLeafInsertedLogs, + processLeafInsertedLogs, processPendingL1ToL2MessageAddedLogs, processTxsPublishedLogs, } from './eth_log_handlers.js'; @@ -209,22 +209,17 @@ export async function retrieveNewL1ToL2Messages( blockUntilSynced: boolean, searchStartBlock: bigint, searchEndBlock: bigint, -): Promise> { - const retrievedNewL1ToL2Messages: [NewInboxLeaf, bigint][] = []; +): Promise> { + const retrievedNewL1ToL2Messages: NewInboxLeaf[] = []; do { if (searchStartBlock > searchEndBlock) { break; } - const leafInsertedLogs = await getLeafInsertedLogs( - publicClient, - newInboxAddress, - searchStartBlock, - searchEndBlock, - ); + const leafInsertedLogs = await getLeafInsertedLogs(publicClient, newInboxAddress, searchStartBlock, searchEndBlock); if (leafInsertedLogs.length === 0) { break; } - const newL1ToL2Messages = processNewLeafInsertedLogs(leafInsertedLogs); + const newL1ToL2Messages = processLeafInsertedLogs(leafInsertedLogs); retrievedNewL1ToL2Messages.push(...newL1ToL2Messages); // handles the case when there are no new messages: searchStartBlock = (leafInsertedLogs.findLast(msgLog => !!msgLog)?.blockNumber || searchStartBlock) + 1n; diff --git a/yarn-project/archiver/src/archiver/eth_log_handlers.ts b/yarn-project/archiver/src/archiver/eth_log_handlers.ts index 19e1009a5e6f..515c8641c0be 100644 --- a/yarn-project/archiver/src/archiver/eth_log_handlers.ts +++ b/yarn-project/archiver/src/archiver/eth_log_handlers.ts @@ -13,30 +13,28 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr } from '@aztec/foundation/fields'; import { BufferReader, numToUInt32BE } from '@aztec/foundation/serialize'; -import { AvailabilityOracleAbi, ContractDeploymentEmitterAbi, InboxAbi, NewInboxAbi, RollupAbi } from '@aztec/l1-artifacts'; +import { + AvailabilityOracleAbi, + ContractDeploymentEmitterAbi, + InboxAbi, + NewInboxAbi, + RollupAbi, +} from '@aztec/l1-artifacts'; import { Hex, Log, PublicClient, decodeFunctionData, getAbiItem, getAddress, hexToBytes } from 'viem'; /** * Processes newly received LeafInserted (L1 to L2) logs. * @param logs - LeafInserted logs. - * @returns Array of all Pending L1 to L2 messages that were processed + * @returns Array of all processed LeafInserted logs */ -export function processNewLeafInsertedLogs( +export function processLeafInsertedLogs( logs: Log[], -): [NewInboxLeaf, bigint][] { - const leaves: [NewInboxLeaf, bigint][] = []; +): NewInboxLeaf[] { + const leaves: NewInboxLeaf[] = []; for (const log of logs) { - const { blockNumber, index, value } = - log.args; - leaves.push([ - new NewInboxLeaf( - blockNumber, - index, - Buffer.from(hexToBytes(value)), - ), - log.blockNumber!, - ]); + const { blockNumber, index, value } = log.args; + leaves.push(new NewInboxLeaf(blockNumber, index, Buffer.from(hexToBytes(value)))); } return leaves; } @@ -389,4 +387,4 @@ export function getLeafInsertedLogs( fromBlock, toBlock: toBlock + 1n, // the toBlock argument in getLogs is exclusive }); -} \ No newline at end of file +} From d84de5e529f744ada9a50e9c13992e9d6991c728 Mon Sep 17 00:00:00 2001 From: benesjan Date: Thu, 7 Mar 2024 12:32:25 +0000 Subject: [PATCH 03/14] WIP --- .../archiver/src/archiver/archiver.ts | 3 +- .../archiver/src/archiver/archiver_store.ts | 3 ++ .../kv_archiver_store/kv_archiver_store.ts | 10 ++++--- .../kv_archiver_store/message_store.ts | 30 +++++++------------ .../l1_to_l2_message_store.ts | 22 +++++++++++++- .../memory_archiver_store.ts | 27 ++++++++++++++++- 6 files changed, 69 insertions(+), 26 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index bc5173193d93..4530ccbdfaf3 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -164,6 +164,7 @@ export class Archiver implements ArchiveSource { if ( currentL1BlockNumber <= lastL1Blocks.addedBlock && + currentL1BlockNumber <= lastL1Blocks.newMessages && currentL1BlockNumber <= lastL1Blocks.addedMessages && currentL1BlockNumber <= lastL1Blocks.cancelledMessages ) { @@ -241,7 +242,7 @@ export class Archiver implements ArchiveSource { this.publicClient, this.inboxAddress, blockUntilSynced, - lastL1Blocks.addedMessages + 1n, + lastL1Blocks.newMessages + 1n, currentL1BlockNumber, ); await this.store.addNewL1ToL2Messages( diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index f6327e053d07..c6121c85aa61 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -23,6 +23,9 @@ import { ContractClassPublic, ContractInstanceWithAddress } from '@aztec/types/c export type ArchiverL1SynchPoint = { /** The last L1 block that added a new L2 block. */ addedBlock: bigint; + /** The last L1 block that added messages from the new inbox. */ + // TODO(#4492): Clean this up and fix the naming + newMessages: bigint; /** The last L1 block that added pending messages */ addedMessages: bigint; /** The last L1 block that cancelled messages */ diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 874894d52878..89dc408e2fad 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -8,6 +8,7 @@ import { L2BlockL2Logs, LogFilter, LogType, + NewInboxLeaf, TxEffect, TxHash, TxReceipt, @@ -151,11 +152,11 @@ export class KVArchiverDataStore implements ArchiverDataStore { /** * Append new L1 to L2 messages to the store. * @param messages - The L1 to L2 messages to be added to the store. - * @param l1BlockNumber - The L1 block number for which to add the messages. + * @param lastMessageL1BlockNumber - The L1 block number in which the last message was emitted. * @returns True if the operation is successful. */ - addNewL1ToL2Messages(messages: L1ToL2Message[], l1BlockNumber: bigint): Promise { - return Promise.resolve(this.#messageStore.addNewL1ToL2Messages(messages, l1BlockNumber)); + addNewL1ToL2Messages(messages: NewInboxLeaf[], lastMessageL1BlockNumber: bigint): Promise { + return Promise.resolve(this.#messageStore.addNewL1ToL2Messages(messages, lastMessageL1BlockNumber)); } /** @@ -300,10 +301,11 @@ export class KVArchiverDataStore implements ArchiverDataStore { */ getL1BlockNumber(): Promise { const addedBlock = this.#blockStore.getL1BlockNumber(); - const { addedMessages, cancelledMessages } = this.#messageStore.getL1BlockNumber(); + const { addedMessages, cancelledMessages, newMessages } = this.#messageStore.getL1BlockNumber(); return Promise.resolve({ addedBlock, addedMessages, + newMessages, cancelledMessages, }); } diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts index 243a12abca0d..4e0554d51a84 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts @@ -1,4 +1,4 @@ -import { L1ToL2Message } from '@aztec/circuit-types'; +import { L1ToL2Message, NewInboxLeaf } from '@aztec/circuit-types'; import { Fr } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { AztecCounter, AztecKVStore, AztecMap, AztecSingleton } from '@aztec/kv-store'; @@ -19,16 +19,18 @@ type Message = { * LMDB implementation of the ArchiverDataStore interface. */ export class MessageStore { - #messages: AztecMap; - #pendingMessagesByFee: AztecCounter<[number, string]>; + #newMessages: AztecMap; #lastL1BlockNewMessages: AztecSingleton; // TODO(#4492): Nuke the following when purging the old inbox + #pendingMessagesByFee: AztecCounter<[number, string]>; + #messages: AztecMap; #lastL1BlockAddingMessages: AztecSingleton; #lastL1BlockCancellingMessages: AztecSingleton; #log = createDebugLogger('aztec:archiver:message_store'); constructor(private db: AztecKVStore) { + this.#newMessages = db.openMap('archiver_l1_to_l2_new_messages'); this.#messages = db.openMap('archiver_l1_to_l2_messages'); this.#pendingMessagesByFee = db.openCounter('archiver_messages_by_fee'); this.#lastL1BlockNewMessages = db.openSingleton('archiver_last_l1_block_new_messages'); @@ -52,31 +54,21 @@ export class MessageStore { /** * Append new L1 to L2 messages to the store. * @param messages - The L1 to L2 messages to be added to the store. - * @param l1BlockNumber - The L1 block number for which to add the messages. + * @param lastMessageL1BlockNumber - The L1 block number in which the last message was emitted. * @returns True if the operation is successful. */ - addNewL1ToL2Messages(messages: L1ToL2Message[], l1BlockNumber: bigint): Promise { + addNewL1ToL2Messages(messages: NewInboxLeaf[], lastMessageL1BlockNumber: bigint): Promise { return this.db.transaction(() => { const lastL1BlockNumber = this.#lastL1BlockAddingMessages.get() ?? 0n; - if (lastL1BlockNumber >= l1BlockNumber) { + if (lastL1BlockNumber >= lastMessageL1BlockNumber) { return false; } - void this.#lastL1BlockNewMessages.set(l1BlockNumber); + void this.#lastL1BlockNewMessages.set(lastMessageL1BlockNumber); for (const message of messages) { - const entryKey = message.entryKey?.toString(); - if (!entryKey) { - throw new Error('Message does not have an entry key'); - } - - void this.#messages.setIfNotExists(entryKey, { - message: message.toBuffer(), - fee: message.fee, - confirmed: false, - }); - - void this.#pendingMessagesByFee.update([message.fee, entryKey], 1); + const key = `${message.blockNumber}-${message.index}`; + void this.#newMessages.setIfNotExists(key, message.leaf); } return true; diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts index e1b0435db8ac..133495691215 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts @@ -1,6 +1,26 @@ -import { L1ToL2Message } from '@aztec/circuit-types'; +import { L1ToL2Message, NewInboxLeaf } from '@aztec/circuit-types'; import { Fr } from '@aztec/foundation/fields'; +/** + * A simple in-memory implementation of an L1 to L2 message store + * that handles message duplication. + * TODO(#4492): Clean this up + */ +export class NewL1ToL2MessageStore { + /** + * A map containing the entry key to the corresponding L1 to L2 + * messages (and the number of times the message has been seen). + */ + protected store: Map = new Map(); + + constructor() {} + + addMessage(message: NewInboxLeaf) { + const key = `${message.blockNumber}-${message.index}`; + this.store.set(key, message.leaf); + } +} + /** * A simple in-memory implementation of an L1 to L2 message store * that handles message duplication. diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index 15dbba4800e9..c3ea10aa4a9b 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -11,6 +11,7 @@ import { LogFilter, LogId, LogType, + NewInboxLeaf, TxEffect, TxHash, TxReceipt, @@ -22,7 +23,7 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { ContractClassPublic, ContractInstanceWithAddress } from '@aztec/types/contracts'; import { ArchiverDataStore } from '../archiver_store.js'; -import { L1ToL2MessageStore, PendingL1ToL2MessageStore } from './l1_to_l2_message_store.js'; +import { L1ToL2MessageStore, NewL1ToL2MessageStore, PendingL1ToL2MessageStore } from './l1_to_l2_message_store.js'; /** * Simple, in-memory implementation of an archiver data store. @@ -65,6 +66,9 @@ export class MemoryArchiverStore implements ArchiverDataStore { */ private extendedContractData: Map = new Map(); + // TODO(#4492): Nuke the other message stores + private newL1ToL2Messages = new NewL1ToL2MessageStore(); + /** * Contains all the confirmed L1 to L2 messages (i.e. messages that were consumed in an L2 block) * It is a map of entryKey to the corresponding L1 to L2 message and the number of times it has appeared @@ -80,6 +84,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { private contractInstances: Map = new Map(); + private lastL1BlockNewMessages: bigint = 0n; private lastL1BlockAddedMessages: bigint = 0n; private lastL1BlockCancelledMessages: bigint = 0n; @@ -173,6 +178,24 @@ export class MemoryArchiverStore implements ArchiverDataStore { return Promise.resolve(true); } + /** + * Append new L1 to L2 messages to the store. + * @param messages - The L1 to L2 messages to be added to the store. + * @param lastMessageL1BlockNumber - The L1 block number in which the last message was emitted. + * @returns True if the operation is successful. + */ + public addNewL1ToL2Messages(messages: NewInboxLeaf[], lastMessageL1BlockNumber: bigint): Promise { + if (lastMessageL1BlockNumber <= this.lastL1BlockAddedMessages) { + return Promise.resolve(false); + } + + this.lastL1BlockAddedMessages = lastMessageL1BlockNumber; + for (const message of messages) { + this.newL1ToL2Messages.addMessage(message); + } + return Promise.resolve(true); + } + /** * Append new pending L1 to L2 messages to the store. * @param messages - The L1 to L2 messages to be added to the store. @@ -494,11 +517,13 @@ export class MemoryArchiverStore implements ArchiverDataStore { public getL1BlockNumber() { const addedBlock = this.l2BlockContexts[this.l2BlockContexts.length - 1]?.block?.getL1BlockNumber() ?? 0n; + const newMessages = this.lastL1BlockNewMessages; const addedMessages = this.lastL1BlockAddedMessages; const cancelledMessages = this.lastL1BlockCancelledMessages; return Promise.resolve({ addedBlock, + newMessages, addedMessages, cancelledMessages, }); From 7eddb74aed88cc05d07b62f06549270d25d2dcf8 Mon Sep 17 00:00:00 2001 From: benesjan Date: Thu, 7 Mar 2024 12:58:20 +0000 Subject: [PATCH 04/14] WIP --- .../archiver/src/archiver/archiver.ts | 9 +++++++ .../archiver/src/archiver/archiver_store.ts | 7 ++++++ .../kv_archiver_store/kv_archiver_store.ts | 13 ++++++++++ .../kv_archiver_store/message_store.ts | 25 ++++++++++++++++++- .../l1_to_l2_message_store.ts | 24 ++++++++++++++++++ .../memory_archiver_store.ts | 9 +++++++ 6 files changed, 86 insertions(+), 1 deletion(-) diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 4530ccbdfaf3..7dfc6b18b7d2 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -600,6 +600,15 @@ export class Archiver implements ArchiveSource { return this.store.getConfirmedL1ToL2Message(entryKey); } + /** + * Gets new L1 to L2 message (to be) included in a given block. + * @param blockNumber - L2 block number to get messages for. + * @returns The L1 to L2 messages/leaves of the messages subtree (throws if not found). + */ + getNewL1ToL2Messages(blockNumber: bigint): Promise { + return this.store.getNewL1ToL2Messages(blockNumber); + } + getContractClassIds(): Promise { return this.store.getContractClassIds(); } diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index c6121c85aa61..1070de6a986f 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -142,6 +142,13 @@ export interface ArchiverDataStore { */ getConfirmedL1ToL2Message(entryKey: Fr): Promise; + /** + * Gets new L1 to L2 message (to be) included in a given block. + * @param blockNumber - L2 block number to get messages for. + * @returns The L1 to L2 messages/leaves of the messages subtree (throws if not found). + */ + getNewL1ToL2Messages(blockNumber: bigint): Promise; + /** * Gets up to `limit` amount of logs starting from `from`. * @param from - Number of the L2 block to which corresponds the first logs to be returned. diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 89dc408e2fad..83191b367894 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -212,6 +212,19 @@ export class KVArchiverDataStore implements ArchiverDataStore { } } + /** + * Gets new L1 to L2 message (to be) included in a given block. + * @param blockNumber - L2 block number to get messages for. + * @returns The L1 to L2 messages/leaves of the messages subtree (throws if not found). + */ + getNewL1ToL2Messages(blockNumber: bigint): Promise { + try { + return Promise.resolve(this.#messageStore.getNewL1ToL2Messages(blockNumber)); + } catch (err) { + return Promise.reject(err); + } + } + /** * Gets up to `limit` amount of logs starting from `from`. * @param start - Number of the L2 block to which corresponds the first logs to be returned. diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts index 4e0554d51a84..7363110e674b 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts @@ -1,5 +1,5 @@ import { L1ToL2Message, NewInboxLeaf } from '@aztec/circuit-types'; -import { Fr } from '@aztec/circuits.js'; +import { Fr, L1_TO_L2_MSG_SUBTREE_HEIGHT } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { AztecCounter, AztecKVStore, AztecMap, AztecSingleton } from '@aztec/kv-store'; @@ -29,6 +29,8 @@ export class MessageStore { #log = createDebugLogger('aztec:archiver:message_store'); + #l1ToL2MessagesSubtreeSize = 2 ** L1_TO_L2_MSG_SUBTREE_HEIGHT; + constructor(private db: AztecKVStore) { this.#newMessages = db.openMap('archiver_l1_to_l2_new_messages'); this.#messages = db.openMap('archiver_l1_to_l2_messages'); @@ -202,4 +204,25 @@ export class MessageStore { return entryKeys; } + + getNewL1ToL2Messages(blockNumber: bigint): Buffer[] { + const messages: Buffer[] = []; + let undefinedMessageFound = false; + for (let messageIndex = 0; messageIndex < this.#l1ToL2MessagesSubtreeSize; messageIndex++) { + // This is inefficient but probably fine for now. + const key = `${blockNumber}-${messageIndex}`; + const message = this.#newMessages.get(key); + if (message) { + if (undefinedMessageFound) { + throw new Error(`Undefined message found in the middle of the messages for block ${blockNumber}`); + } + messages.push(message); + } else { + undefinedMessageFound = true; + // We continue iterating over messages here to verify that there are no more messages after the undefined one. + // --> If this was the case this would imply there is some issue with log fetching. + } + } + return messages; + } } diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts index 133495691215..2146d05b74fd 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts @@ -1,4 +1,5 @@ import { L1ToL2Message, NewInboxLeaf } from '@aztec/circuit-types'; +import { L1_TO_L2_MSG_SUBTREE_HEIGHT } from '@aztec/circuits.js/constants'; import { Fr } from '@aztec/foundation/fields'; /** @@ -13,12 +14,35 @@ export class NewL1ToL2MessageStore { */ protected store: Map = new Map(); + #l1ToL2MessagesSubtreeSize = 2 ** L1_TO_L2_MSG_SUBTREE_HEIGHT; + constructor() {} addMessage(message: NewInboxLeaf) { const key = `${message.blockNumber}-${message.index}`; this.store.set(key, message.leaf); } + + getMessages(blockNumber: bigint): Buffer[] { + const messages: Buffer[] = []; + let undefinedMessageFound = false; + for (let messageIndex = 0; messageIndex < this.#l1ToL2MessagesSubtreeSize; messageIndex++) { + // This is inefficient but probably fine for now. + const key = `${blockNumber}-${messageIndex}`; + const message = this.store.get(key); + if (message) { + if (undefinedMessageFound) { + throw new Error(`Undefined message found in the middle of the messages for block ${blockNumber}`); + } + messages.push(message); + } else { + undefinedMessageFound = true; + // We continue iterating over messages here to verify that there are no more messages after the undefined one. + // --> If this was the case this would imply there is some issue with log fetching. + } + } + return messages; + } } /** diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index c3ea10aa4a9b..d9bf07ebb308 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -344,6 +344,15 @@ export class MemoryArchiverStore implements ArchiverDataStore { return Promise.resolve(message); } + /** + * Gets new L1 to L2 message (to be) included in a given block. + * @param blockNumber - L2 block number to get messages for. + * @returns The L1 to L2 messages/leaves of the messages subtree (throws if not found). + */ + getNewL1ToL2Messages(blockNumber: bigint): Promise { + return Promise.resolve(this.newL1ToL2Messages.getMessages(blockNumber)); + } + /** * Gets up to `limit` amount of logs starting from `from`. * @param from - Number of the L2 block to which corresponds the first logs to be returned. From ea861adacfe7b4369aac694e35e99d7e4d13cc7d Mon Sep 17 00:00:00 2001 From: benesjan Date: Thu, 7 Mar 2024 14:20:32 +0000 Subject: [PATCH 05/14] WIP --- yarn-project/archiver/src/archiver/archiver.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 7dfc6b18b7d2..fcc251e4c386 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -73,6 +73,7 @@ export class Archiver implements ArchiveSource { * @param publicClient - A client for interacting with the Ethereum node. * @param rollupAddress - Ethereum address of the rollup contract. * @param inboxAddress - Ethereum address of the inbox contract. + * @param newInboxAddress - Ethereum address of the new inbox contract. * @param registryAddress - Ethereum address of the registry contract. * @param contractDeploymentEmitterAddress - Ethereum address of the contractDeploymentEmitter contract. * @param pollingIntervalMs - The interval for polling for L1 logs (in milliseconds). @@ -84,6 +85,7 @@ export class Archiver implements ArchiveSource { private readonly rollupAddress: EthAddress, private readonly availabilityOracleAddress: EthAddress, private readonly inboxAddress: EthAddress, + private readonly newInboxAddress: EthAddress, private readonly registryAddress: EthAddress, private readonly contractDeploymentEmitterAddress: EthAddress, private readonly store: ArchiverDataStore, @@ -115,6 +117,7 @@ export class Archiver implements ArchiveSource { config.l1Contracts.rollupAddress, config.l1Contracts.availabilityOracleAddress, config.l1Contracts.inboxAddress, + config.l1Contracts.newInboxAddress, config.l1Contracts.registryAddress, config.l1Contracts.contractDeploymentEmitterAddress, archiverStore, @@ -240,7 +243,7 @@ export class Archiver implements ArchiveSource { const retrievedNewL1ToL2Messages = await retrieveNewL1ToL2Messages( this.publicClient, - this.inboxAddress, + this.newInboxAddress, blockUntilSynced, lastL1Blocks.newMessages + 1n, currentL1BlockNumber, From 7e4608d20847b588829d1c2c876bbba4f4ae37a9 Mon Sep 17 00:00:00 2001 From: benesjan Date: Thu, 7 Mar 2024 15:29:44 +0000 Subject: [PATCH 06/14] injecting in new inbox address ugly way --- .../archiver/src/archiver/archiver.test.ts | 28 +++++++++++++++++-- .../archiver/src/archiver/archiver.ts | 16 +++++++++-- yarn-project/archiver/src/index.ts | 16 ++++++++++- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver.test.ts b/yarn-project/archiver/src/archiver/archiver.test.ts index 6f5513b228fa..b7fe24bc8d25 100644 --- a/yarn-project/archiver/src/archiver/archiver.test.ts +++ b/yarn-project/archiver/src/archiver/archiver.test.ts @@ -6,7 +6,17 @@ import { sleep } from '@aztec/foundation/sleep'; import { AvailabilityOracleAbi, ContractDeploymentEmitterAbi, InboxAbi, RollupAbi } from '@aztec/l1-artifacts'; import { MockProxy, mock } from 'jest-mock-extended'; -import { Chain, HttpTransport, Log, PublicClient, Transaction, encodeFunctionData, toHex } from 'viem'; +import { + Chain, + HttpTransport, + Log, + PublicClient, + Transaction, + encodeFunctionData, + getAddress, + getContract, + toHex, +} from 'viem'; import { Archiver } from './archiver.js'; import { ArchiverDataStore } from './archiver_store.js'; @@ -21,10 +31,21 @@ describe('Archiver', () => { const blockNumbers = [1, 2, 3]; let publicClient: MockProxy>; let archiverStore: ArchiverDataStore; + let newInboxAddress!: EthAddress; - beforeEach(() => { + beforeEach(async () => { publicClient = mock>(); archiverStore = new MemoryArchiverStore(1000); + + // TODO(#4492): Nuke this once the old inbox is purged + { + const rollup = getContract({ + address: getAddress(rollupAddress.toString()), + abi: RollupAbi, + client: publicClient, + }); + newInboxAddress = EthAddress.fromString(await rollup.read.NEW_INBOX()); + } }); it('can start, sync and stop and handle l1 to l2 messages and logs', async () => { @@ -33,6 +54,7 @@ describe('Archiver', () => { EthAddress.fromString(rollupAddress), EthAddress.fromString(availabilityOracleAddress), EthAddress.fromString(inboxAddress), + newInboxAddress, EthAddress.fromString(registryAddress), EthAddress.fromString(contractDeploymentEmitterAddress), archiverStore, @@ -148,6 +170,7 @@ describe('Archiver', () => { EthAddress.fromString(rollupAddress), EthAddress.fromString(availabilityOracleAddress), EthAddress.fromString(inboxAddress), + newInboxAddress, EthAddress.fromString(registryAddress), EthAddress.fromString(contractDeploymentEmitterAddress), archiverStore, @@ -232,6 +255,7 @@ describe('Archiver', () => { EthAddress.fromString(rollupAddress), EthAddress.fromString(availabilityOracleAddress), EthAddress.fromString(inboxAddress), + newInboxAddress, EthAddress.fromString(registryAddress), EthAddress.fromString(contractDeploymentEmitterAddress), archiverStore, diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index fcc251e4c386..bda21f4a5bdf 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -25,6 +25,7 @@ import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr } from '@aztec/foundation/fields'; import { DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; +import { RollupAbi } from '@aztec/l1-artifacts'; import { ClassRegistererAddress } from '@aztec/protocol-contracts/class-registerer'; import { InstanceDeployerAddress } from '@aztec/protocol-contracts/instance-deployer'; import { @@ -34,7 +35,7 @@ import { ContractInstanceWithAddress, } from '@aztec/types/contracts'; -import { Chain, HttpTransport, PublicClient, createPublicClient, http } from 'viem'; +import { Chain, HttpTransport, PublicClient, createPublicClient, getAddress, getContract, http } from 'viem'; import { ArchiverDataStore } from './archiver_store.js'; import { ArchiverConfig } from './config.js'; @@ -112,12 +113,23 @@ export class Archiver implements ArchiveSource { pollingInterval: config.viemPollingIntervalMS, }); + // TODO(#4492): Nuke this once the old inbox is purged + let newInboxAddress!: EthAddress; + { + const rollup = getContract({ + address: getAddress(config.l1Contracts.rollupAddress.toString()), + abi: RollupAbi, + client: publicClient, + }); + newInboxAddress = EthAddress.fromString(await rollup.read.NEW_INBOX()); + } + const archiver = new Archiver( publicClient, config.l1Contracts.rollupAddress, config.l1Contracts.availabilityOracleAddress, config.l1Contracts.inboxAddress, - config.l1Contracts.newInboxAddress, + newInboxAddress, config.l1Contracts.registryAddress, config.l1Contracts.contractDeploymentEmitterAddress, archiverStore, diff --git a/yarn-project/archiver/src/index.ts b/yarn-project/archiver/src/index.ts index 7f306cdf6342..f6e3c1003e4f 100644 --- a/yarn-project/archiver/src/index.ts +++ b/yarn-project/archiver/src/index.ts @@ -1,7 +1,9 @@ +import { EthAddress } from '@aztec/foundation/eth-address'; import { createDebugLogger } from '@aztec/foundation/log'; import { fileURLToPath } from '@aztec/foundation/url'; +import { RollupAbi } from '@aztec/l1-artifacts'; -import { createPublicClient, http } from 'viem'; +import { createPublicClient, getAddress, getContract, http } from 'viem'; import { localhost } from 'viem/chains'; import { Archiver, getConfigEnvVars } from './archiver/index.js'; @@ -27,11 +29,23 @@ async function main() { const archiverStore = new MemoryArchiverStore(1000); + // TODO(#4492): Nuke this once the old inbox is purged + let newInboxAddress!: EthAddress; + { + const rollup = getContract({ + address: getAddress(l1Contracts.rollupAddress.toString()), + abi: RollupAbi, + client: publicClient, + }); + newInboxAddress = EthAddress.fromString(await rollup.read.NEW_INBOX()); + } + const archiver = new Archiver( publicClient, l1Contracts.rollupAddress, l1Contracts.availabilityOracleAddress, l1Contracts.inboxAddress, + newInboxAddress, l1Contracts.registryAddress, l1Contracts.contractDeploymentEmitterAddress, archiverStore, From 0225cb4d9c8b1642e1515f87228a082b32bd37fb Mon Sep 17 00:00:00 2001 From: benesjan Date: Thu, 7 Mar 2024 16:35:04 +0000 Subject: [PATCH 07/14] WIP --- .../archiver/src/archiver/archiver.test.ts | 97 ++++++++++--------- .../src/archiver/archiver_store_test_suite.ts | 14 +++ .../memory_archiver_store.ts | 2 +- 3 files changed, 68 insertions(+), 45 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver.test.ts b/yarn-project/archiver/src/archiver/archiver.test.ts index b7fe24bc8d25..aa074dac5c53 100644 --- a/yarn-project/archiver/src/archiver/archiver.test.ts +++ b/yarn-project/archiver/src/archiver/archiver.test.ts @@ -3,60 +3,47 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr } from '@aztec/foundation/fields'; import { sleep } from '@aztec/foundation/sleep'; -import { AvailabilityOracleAbi, ContractDeploymentEmitterAbi, InboxAbi, RollupAbi } from '@aztec/l1-artifacts'; +import { + AvailabilityOracleAbi, + ContractDeploymentEmitterAbi, + InboxAbi, + NewInboxAbi, + RollupAbi, +} from '@aztec/l1-artifacts'; import { MockProxy, mock } from 'jest-mock-extended'; -import { - Chain, - HttpTransport, - Log, - PublicClient, - Transaction, - encodeFunctionData, - getAddress, - getContract, - toHex, -} from 'viem'; +import { Chain, HttpTransport, Log, PublicClient, Transaction, encodeFunctionData, toHex } from 'viem'; import { Archiver } from './archiver.js'; import { ArchiverDataStore } from './archiver_store.js'; import { MemoryArchiverStore } from './memory_archiver_store/memory_archiver_store.js'; describe('Archiver', () => { - const rollupAddress = EthAddress.ZERO.toString(); - const inboxAddress = EthAddress.ZERO.toString(); - const registryAddress = EthAddress.ZERO.toString(); - const availabilityOracleAddress = EthAddress.ZERO.toString(); - const contractDeploymentEmitterAddress = '0x0000000000000000000000000000000000000001'; + const rollupAddress = EthAddress.ZERO; + const inboxAddress = EthAddress.ZERO; + // TODO(#4492): Nuke this once the old inbox is purged + const newInboxAddress = EthAddress.ZERO; + const registryAddress = EthAddress.ZERO; + const availabilityOracleAddress = EthAddress.ZERO; + const contractDeploymentEmitterAddress = EthAddress.fromString('0x0000000000000000000000000000000000000001'); const blockNumbers = [1, 2, 3]; let publicClient: MockProxy>; let archiverStore: ArchiverDataStore; - let newInboxAddress!: EthAddress; - beforeEach(async () => { + beforeEach(() => { publicClient = mock>(); archiverStore = new MemoryArchiverStore(1000); - - // TODO(#4492): Nuke this once the old inbox is purged - { - const rollup = getContract({ - address: getAddress(rollupAddress.toString()), - abi: RollupAbi, - client: publicClient, - }); - newInboxAddress = EthAddress.fromString(await rollup.read.NEW_INBOX()); - } }); it('can start, sync and stop and handle l1 to l2 messages and logs', async () => { const archiver = new Archiver( publicClient, - EthAddress.fromString(rollupAddress), - EthAddress.fromString(availabilityOracleAddress), - EthAddress.fromString(inboxAddress), + rollupAddress, + availabilityOracleAddress, + inboxAddress, newInboxAddress, - EthAddress.fromString(registryAddress), - EthAddress.fromString(contractDeploymentEmitterAddress), + registryAddress, + contractDeploymentEmitterAddress, archiverStore, 1000, ); @@ -104,11 +91,13 @@ describe('Archiver', () => { publicClient.getLogs .mockResolvedValueOnce(l1ToL2MessageAddedEvents.slice(0, 2).flat()) .mockResolvedValueOnce([]) // no messages to cancel + .mockResolvedValueOnce([makeLeafInsertedEvent(98n, 1n, 0n), makeLeafInsertedEvent(99n, 1n, 1n)]) .mockResolvedValueOnce([makeTxsPublishedEvent(101n, blocks[0].body.getTxsEffectsHash())]) .mockResolvedValueOnce([makeL2BlockProcessedEvent(101n, 1n)]) .mockResolvedValueOnce([makeContractDeploymentEvent(103n, blocks[0])]) // the first loop of the archiver ends here at block 2500 .mockResolvedValueOnce(l1ToL2MessageAddedEvents.slice(2, 4).flat()) .mockResolvedValueOnce(makeL1ToL2MessageCancelledEvents(2503n, l1ToL2MessagesToCancel)) + .mockResolvedValueOnce([makeLeafInsertedEvent(2504n, 2n, 0n), makeLeafInsertedEvent(2504n, 2n, 1n)]) .mockResolvedValueOnce([ makeTxsPublishedEvent(2510n, blocks[1].body.getTxsEffectsHash()), makeTxsPublishedEvent(2520n, blocks[2].body.getTxsEffectsHash()), @@ -167,12 +156,12 @@ describe('Archiver', () => { const numL2BlocksInTest = 2; const archiver = new Archiver( publicClient, - EthAddress.fromString(rollupAddress), - EthAddress.fromString(availabilityOracleAddress), - EthAddress.fromString(inboxAddress), + rollupAddress, + availabilityOracleAddress, + inboxAddress, newInboxAddress, - EthAddress.fromString(registryAddress), - EthAddress.fromString(contractDeploymentEmitterAddress), + registryAddress, + contractDeploymentEmitterAddress, archiverStore, 1000, ); @@ -222,6 +211,7 @@ describe('Archiver', () => { ); }) .mockResolvedValueOnce([]) + .mockResolvedValueOnce([makeLeafInsertedEvent(66n, 1n, 0n), makeLeafInsertedEvent(68n, 1n, 1n)]) .mockResolvedValueOnce([ makeTxsPublishedEvent(70n, blocks[0].body.getTxsEffectsHash()), makeTxsPublishedEvent(80n, blocks[1].body.getTxsEffectsHash()), @@ -252,12 +242,12 @@ describe('Archiver', () => { it('pads L1 to L2 messages', async () => { const archiver = new Archiver( publicClient, - EthAddress.fromString(rollupAddress), - EthAddress.fromString(availabilityOracleAddress), - EthAddress.fromString(inboxAddress), + rollupAddress, + availabilityOracleAddress, + inboxAddress, newInboxAddress, - EthAddress.fromString(registryAddress), - EthAddress.fromString(contractDeploymentEmitterAddress), + registryAddress, + contractDeploymentEmitterAddress, archiverStore, 1000, ); @@ -279,6 +269,7 @@ describe('Archiver', () => { ), ) .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]) .mockResolvedValueOnce([makeTxsPublishedEvent(101n, block.body.getTxsEffectsHash())]) .mockResolvedValueOnce([makeL2BlockProcessedEvent(101n, 1n)]) .mockResolvedValue([]); @@ -402,6 +393,24 @@ function makeL1ToL2MessageCancelledEvents(l1BlockNum: bigint, entryKeys: string[ }); } +/** + * Makes fake L1ToL2 LeafInserted events for testing purposes. + * @param l1BlockNum - L1 block number. + * @param l2BlockNumber - The L2 block number of the leaf inserted. + * @returns LeafInserted event logs. + */ +function makeLeafInsertedEvent(l1BlockNum: bigint, l2BlockNumber: bigint, index: bigint) { + return { + blockNumber: l1BlockNum, + args: { + blockNumber: l2BlockNumber, + index, + value: Fr.random().toString(), + }, + transactionHash: `0x${l1BlockNum}`, + } as Log; +} + /** * Makes a fake rollup tx for testing purposes. * @param block - The L2Block. diff --git a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts index 98261dfdd633..40a8178109e7 100644 --- a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts +++ b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts @@ -5,6 +5,7 @@ import { L2BlockContext, LogId, LogType, + NewInboxLeaf, TxHash, UnencryptedL2Log, } from '@aztec/circuit-types'; @@ -96,6 +97,7 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch addedBlock: 0n, addedMessages: 0n, cancelledMessages: 0n, + newMessages: 0n, }); }); @@ -105,6 +107,7 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch addedBlock: blocks.at(-1)!.getL1BlockNumber(), addedMessages: 0n, cancelledMessages: 0n, + newMessages: 0n, }); }); @@ -114,6 +117,16 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch addedBlock: 0n, addedMessages: 1n, cancelledMessages: 0n, + newMessages: 0n, + }); + }); + it('returns the L1 block number that most recently added messages from new inbox', async () => { + await store.addNewL1ToL2Messages([new NewInboxLeaf(0n, 0n, Buffer.alloc(32))], 1n); + await expect(store.getL1BlockNumber()).resolves.toEqual({ + addedBlock: 0n, + addedMessages: 0n, + cancelledMessages: 0n, + newMessages: 1n, }); }); it('returns the L1 block number that most recently cancelled pending messages', async () => { @@ -124,6 +137,7 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch addedBlock: 0n, addedMessages: 1n, cancelledMessages: 2n, + newMessages: 0n, }); }); }); diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index d9bf07ebb308..c55e82206b2f 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -189,7 +189,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { return Promise.resolve(false); } - this.lastL1BlockAddedMessages = lastMessageL1BlockNumber; + this.lastL1BlockNewMessages = lastMessageL1BlockNumber; for (const message of messages) { this.newL1ToL2Messages.addMessage(message); } From 2bd8f8777d4f873468db0bf02bf2485e86c71ded Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 8 Mar 2024 08:19:52 +0000 Subject: [PATCH 08/14] WIP on tests --- .../src/archiver/archiver_store_test_suite.ts | 26 ++++++++++++++++++- .../kv_archiver_store/message_store.ts | 3 +++ .../docs/rollup-circuits/root-rollup.md | 4 +-- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts index 40a8178109e7..c08730d23ce5 100644 --- a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts +++ b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts @@ -10,7 +10,7 @@ import { UnencryptedL2Log, } from '@aztec/circuit-types'; import '@aztec/circuit-types/jest'; -import { AztecAddress, Fr, INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js'; +import { AztecAddress, Fr, INITIAL_L2_BLOCK_NUM, L1_TO_L2_MSG_SUBTREE_HEIGHT } from '@aztec/circuits.js'; import { makeContractClassPublic } from '@aztec/circuits.js/testing'; import { randomBytes } from '@aztec/foundation/crypto'; import { ContractClassPublic, ContractInstanceWithAddress, SerializableContractInstance } from '@aztec/types/contracts'; @@ -223,6 +223,30 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); }); + describe('getNewL1ToL2Messages', () => { + const l1ToL2MessageSubtreeSize = 2 ** L1_TO_L2_MSG_SUBTREE_HEIGHT; + + const generateBlockMessages = (blockNumber: bigint, numMessages: number) => + Array.from({ length: numMessages }, (_, i) => new NewInboxLeaf(blockNumber, BigInt(i), randomBytes(32))); + + it('returns messages in correct order', async () => { + const l2BlockNumber = 13n; + + const msgs = generateBlockMessages(l2BlockNumber, l1ToL2MessageSubtreeSize); + const shuffledMessages = msgs.slice().sort(() => Math.random() - 0.5); + await store.addNewL1ToL2Messages(shuffledMessages, 100n); + const retrievedMessages = await store.getNewL1ToL2Messages(l2BlockNumber); + + const expectedLeavesOrder = msgs.map(msg => msg.leaf); + expect(expectedLeavesOrder).toEqual(retrievedMessages); + }); + + it('throws an error if messages are sequenced incorrectly', async () => { + const msgs = [new NewInboxLeaf(1n, 0n, randomBytes(32)), new NewInboxLeaf(1n, 1n, randomBytes(32))]; + await store.addNewL1ToL2Messages(msgs, 100n); + }); + }); + describe('getPendingL1ToL2EntryKeys', () => { it('returns previously stored pending L1 to L2 messages', async () => { const message = L1ToL2Message.random(Fr.random()); diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts index 7363110e674b..e44295ec2faa 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts @@ -69,6 +69,9 @@ export class MessageStore { void this.#lastL1BlockNewMessages.set(lastMessageL1BlockNumber); for (const message of messages) { + if (message.index >= this.#l1ToL2MessagesSubtreeSize) { + throw new Error(`Message index ${message.index} is out of range`); + } const key = `${message.blockNumber}-${message.index}`; void this.#newMessages.setIfNotExists(key, message.leaf); } diff --git a/yellow-paper/docs/rollup-circuits/root-rollup.md b/yellow-paper/docs/rollup-circuits/root-rollup.md index 712b4769170d..3cdfa06d6737 100644 --- a/yellow-paper/docs/rollup-circuits/root-rollup.md +++ b/yellow-paper/docs/rollup-circuits/root-rollup.md @@ -210,8 +210,8 @@ def RootRollupCircuit( parent.state.l1_to_l2_message_tree, l1_to_l2_roots.public_inputs.converted_root, l1_to_l2_msgs_sibling_path, - L1_TO_L2_SUBTREE_HEIGHT, - L1_To_L2_HEIGHT + L1_TO_L2_MSG_SUBTREE_HEIGHT, + L1_TO_L2_MSG_TREE_HEIGHT ) header = Header( From 43915714aea0debdb78ba355a6a636e1e27ad6b3 Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 8 Mar 2024 08:46:56 +0000 Subject: [PATCH 09/14] WIP --- .../src/archiver/archiver_store_test_suite.ts | 26 +++++++++++++++++-- .../kv_archiver_store/message_store.ts | 2 +- .../l1_to_l2_message_store.ts | 3 +++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts index c08730d23ce5..86018407514e 100644 --- a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts +++ b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts @@ -241,9 +241,31 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch expect(expectedLeavesOrder).toEqual(retrievedMessages); }); - it('throws an error if messages are sequenced incorrectly', async () => { - const msgs = [new NewInboxLeaf(1n, 0n, randomBytes(32)), new NewInboxLeaf(1n, 1n, randomBytes(32))]; + it('throws if it is impossible to sequence messages correctly', async () => { + const l2BlockNumber = 13n; + + const msgs = generateBlockMessages(l2BlockNumber, l1ToL2MessageSubtreeSize - 1); + // We replace a message with index 4 with a message with index at the end of the tree + // --> with that there will be a gap and it will be impossible to sequence the messages + msgs[4] = new NewInboxLeaf(l2BlockNumber, BigInt(l1ToL2MessageSubtreeSize - 1), randomBytes(32)); + await store.addNewL1ToL2Messages(msgs, 100n); + await expect(store.getNewL1ToL2Messages(l2BlockNumber)).rejects.toThrowError( + `Undefined message found in the middle of the messages for block ${l2BlockNumber}`, + ); + }); + + it('throws if adding more messages than fits into a block', async () => { + const l2BlockNumber = 13n; + + const msgs = generateBlockMessages(l2BlockNumber, l1ToL2MessageSubtreeSize + 1); + // We replace a message with index 4 with a message with index at the end of the tree + // --> with that there will be a gap and it will be impossible to sequence the messages + msgs[4] = new NewInboxLeaf(l2BlockNumber, BigInt(l1ToL2MessageSubtreeSize - 1), randomBytes(32)); + + await expect(store.addNewL1ToL2Messages(msgs, 100n)).rejects.toThrowError( + `Message index ${l1ToL2MessageSubtreeSize} out of subtree range`, + ); }); }); diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts index e44295ec2faa..94d28bbff4ee 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts @@ -70,7 +70,7 @@ export class MessageStore { for (const message of messages) { if (message.index >= this.#l1ToL2MessagesSubtreeSize) { - throw new Error(`Message index ${message.index} is out of range`); + throw new Error(`Message index ${message.index} out of subtree range`); } const key = `${message.blockNumber}-${message.index}`; void this.#newMessages.setIfNotExists(key, message.leaf); diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts index 2146d05b74fd..f6e3537c2856 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts @@ -19,6 +19,9 @@ export class NewL1ToL2MessageStore { constructor() {} addMessage(message: NewInboxLeaf) { + if (message.index >= this.#l1ToL2MessagesSubtreeSize) { + throw new Error(`Message index ${message.index} out of subtree range`); + } const key = `${message.blockNumber}-${message.index}`; this.store.set(key, message.leaf); } From 8ea370d55edb38902dc24b6946d5441182676a64 Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 8 Mar 2024 09:11:01 +0000 Subject: [PATCH 10/14] WIP --- .../src/archiver/archiver_store_test_suite.ts | 25 +++++++------------ .../kv_archiver_store/message_store.ts | 2 +- .../l1_to_l2_message_store.ts | 2 +- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts index 86018407514e..53280f53cfc0 100644 --- a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts +++ b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts @@ -223,15 +223,15 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); }); - describe('getNewL1ToL2Messages', () => { + // TODO(#4492): Drop the "New" bellow once the old inbox is purged + describe('New L1 to L2 Messages', () => { + const l2BlockNumber = 13n; const l1ToL2MessageSubtreeSize = 2 ** L1_TO_L2_MSG_SUBTREE_HEIGHT; const generateBlockMessages = (blockNumber: bigint, numMessages: number) => Array.from({ length: numMessages }, (_, i) => new NewInboxLeaf(blockNumber, BigInt(i), randomBytes(32))); it('returns messages in correct order', async () => { - const l2BlockNumber = 13n; - const msgs = generateBlockMessages(l2BlockNumber, l1ToL2MessageSubtreeSize); const shuffledMessages = msgs.slice().sort(() => Math.random() - 0.5); await store.addNewL1ToL2Messages(shuffledMessages, 100n); @@ -242,30 +242,23 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); it('throws if it is impossible to sequence messages correctly', async () => { - const l2BlockNumber = 13n; - const msgs = generateBlockMessages(l2BlockNumber, l1ToL2MessageSubtreeSize - 1); // We replace a message with index 4 with a message with index at the end of the tree // --> with that there will be a gap and it will be impossible to sequence the messages msgs[4] = new NewInboxLeaf(l2BlockNumber, BigInt(l1ToL2MessageSubtreeSize - 1), randomBytes(32)); await store.addNewL1ToL2Messages(msgs, 100n); - await expect(store.getNewL1ToL2Messages(l2BlockNumber)).rejects.toThrowError( - `Undefined message found in the middle of the messages for block ${l2BlockNumber}`, - ); + await expect(async () => { + await store.getNewL1ToL2Messages(l2BlockNumber); + }).rejects.toThrow(`L1 to L2 message gap found in block ${l2BlockNumber}`); }); it('throws if adding more messages than fits into a block', async () => { - const l2BlockNumber = 13n; - const msgs = generateBlockMessages(l2BlockNumber, l1ToL2MessageSubtreeSize + 1); - // We replace a message with index 4 with a message with index at the end of the tree - // --> with that there will be a gap and it will be impossible to sequence the messages - msgs[4] = new NewInboxLeaf(l2BlockNumber, BigInt(l1ToL2MessageSubtreeSize - 1), randomBytes(32)); - await expect(store.addNewL1ToL2Messages(msgs, 100n)).rejects.toThrowError( - `Message index ${l1ToL2MessageSubtreeSize} out of subtree range`, - ); + await expect(async () => { + await store.addNewL1ToL2Messages(msgs, 100n); + }).rejects.toThrow(`Message index ${l1ToL2MessageSubtreeSize} out of subtree range`); }); }); diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts index 94d28bbff4ee..2ca9258316b5 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts @@ -217,7 +217,7 @@ export class MessageStore { const message = this.#newMessages.get(key); if (message) { if (undefinedMessageFound) { - throw new Error(`Undefined message found in the middle of the messages for block ${blockNumber}`); + throw new Error(`L1 to L2 message gap found in block ${blockNumber}`); } messages.push(message); } else { diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts index f6e3537c2856..55ba69acfa01 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts @@ -35,7 +35,7 @@ export class NewL1ToL2MessageStore { const message = this.store.get(key); if (message) { if (undefinedMessageFound) { - throw new Error(`Undefined message found in the middle of the messages for block ${blockNumber}`); + throw new Error(`L1 to L2 message gap found in block ${blockNumber}`); } messages.push(message); } else { From 8d21119d23266ef47c9db8ce9ba6a5d686762764 Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 8 Mar 2024 09:53:18 +0000 Subject: [PATCH 11/14] more tests + fix --- .../archiver/src/archiver/archiver.test.ts | 23 ++++++++++++++++++- .../memory_archiver_store.ts | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver.test.ts b/yarn-project/archiver/src/archiver/archiver.test.ts index aa074dac5c53..7bdfe6516399 100644 --- a/yarn-project/archiver/src/archiver/archiver.test.ts +++ b/yarn-project/archiver/src/archiver/archiver.test.ts @@ -97,7 +97,12 @@ describe('Archiver', () => { .mockResolvedValueOnce([makeContractDeploymentEvent(103n, blocks[0])]) // the first loop of the archiver ends here at block 2500 .mockResolvedValueOnce(l1ToL2MessageAddedEvents.slice(2, 4).flat()) .mockResolvedValueOnce(makeL1ToL2MessageCancelledEvents(2503n, l1ToL2MessagesToCancel)) - .mockResolvedValueOnce([makeLeafInsertedEvent(2504n, 2n, 0n), makeLeafInsertedEvent(2504n, 2n, 1n)]) + .mockResolvedValueOnce([ + makeLeafInsertedEvent(2504n, 2n, 0n), + makeLeafInsertedEvent(2505n, 2n, 1n), + makeLeafInsertedEvent(2505n, 2n, 2n), + makeLeafInsertedEvent(2506n, 3n, 1n), + ]) .mockResolvedValueOnce([ makeTxsPublishedEvent(2510n, blocks[1].body.getTxsEffectsHash()), makeTxsPublishedEvent(2520n, blocks[2].body.getTxsEffectsHash()), @@ -121,6 +126,22 @@ describe('Archiver', () => { latestBlockNum = await archiver.getBlockNumber(); expect(latestBlockNum).toEqual(3); + // New L1 to L2 messages + { + // Checks that I get correct amount of sequenced new messages for L2 blocks 1 and 2 + let newL1ToL2Messages = await archiver.getNewL1ToL2Messages(1n); + expect(newL1ToL2Messages.length).toEqual(2); + + newL1ToL2Messages = await archiver.getNewL1ToL2Messages(2n); + expect(newL1ToL2Messages.length).toEqual(2); + + // Check that I cannot get messages for block 3 because there is a message gap (message with index 0 was not + // processed) + await expect(async () => { + await archiver.getNewL1ToL2Messages(3n); + }).rejects.toThrow(`L1 to L2 message gap found in block ${3}`); + } + // Check that only 2 messages (l1ToL2MessageAddedEvents[3][2] and l1ToL2MessageAddedEvents[3][3]) are pending. // Other two (l1ToL2MessageAddedEvents[3][0..2]) were cancelled. And the previous messages were confirmed. const expectedPendingEntryKeys = [ diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index c55e82206b2f..d299a7d99594 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -185,7 +185,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { * @returns True if the operation is successful. */ public addNewL1ToL2Messages(messages: NewInboxLeaf[], lastMessageL1BlockNumber: bigint): Promise { - if (lastMessageL1BlockNumber <= this.lastL1BlockAddedMessages) { + if (lastMessageL1BlockNumber <= this.lastL1BlockNewMessages) { return Promise.resolve(false); } From eddd3ae9bfe35518b07730261834d79c0d558389 Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 8 Mar 2024 10:01:02 +0000 Subject: [PATCH 12/14] fix --- .../archiver/src/archiver/kv_archiver_store/message_store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts index 2ca9258316b5..a56dd2c30506 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts @@ -61,7 +61,7 @@ export class MessageStore { */ addNewL1ToL2Messages(messages: NewInboxLeaf[], lastMessageL1BlockNumber: bigint): Promise { return this.db.transaction(() => { - const lastL1BlockNumber = this.#lastL1BlockAddingMessages.get() ?? 0n; + const lastL1BlockNumber = this.#lastL1BlockNewMessages.get() ?? 0n; if (lastL1BlockNumber >= lastMessageL1BlockNumber) { return false; } From fd23e3b4b6d8c5d991434a01096b3127f5facb42 Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 8 Mar 2024 10:17:11 +0000 Subject: [PATCH 13/14] test fix --- yarn-project/archiver/src/archiver/archiver.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/archiver/src/archiver/archiver.test.ts b/yarn-project/archiver/src/archiver/archiver.test.ts index 7bdfe6516399..7fd15cf32cf9 100644 --- a/yarn-project/archiver/src/archiver/archiver.test.ts +++ b/yarn-project/archiver/src/archiver/archiver.test.ts @@ -133,7 +133,7 @@ describe('Archiver', () => { expect(newL1ToL2Messages.length).toEqual(2); newL1ToL2Messages = await archiver.getNewL1ToL2Messages(2n); - expect(newL1ToL2Messages.length).toEqual(2); + expect(newL1ToL2Messages.length).toEqual(3); // Check that I cannot get messages for block 3 because there is a message gap (message with index 0 was not // processed) From 2f28e89d1166113c5e7a51a80e9383182af54964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Bene=C5=A1?= Date: Mon, 11 Mar 2024 09:04:34 +0100 Subject: [PATCH 14/14] Update yarn-project/archiver/src/archiver/archiver_store_test_suite.ts Co-authored-by: esau <152162806+sklppy88@users.noreply.github.com> --- yarn-project/archiver/src/archiver/archiver_store_test_suite.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts index 53280f53cfc0..ef3e28a0f7f0 100644 --- a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts +++ b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts @@ -223,7 +223,7 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); }); - // TODO(#4492): Drop the "New" bellow once the old inbox is purged + // TODO(#4492): Drop the "New" below once the old inbox is purged describe('New L1 to L2 Messages', () => { const l2BlockNumber = 13n; const l1ToL2MessageSubtreeSize = 2 ** L1_TO_L2_MSG_SUBTREE_HEIGHT;