feat: add api for inclusion proof of outgoing message in block #4562

.circleci/config.yml

@@ -880,6 +880,20 @@ jobs:
           aztec_manifest_key: end-to-end
     <<: *defaults_e2e_test
 
+
+  e2e-outbox:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Test"
+          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_outbox.test.ts
+          aztec_manifest_key: end-to-end
+    <<: *defaults_e2e_test
+
   uniswap-trade-on-l1-from-l2:
     steps:
       - *checkout
@@ -1412,6 +1426,7 @@ workflows:
       - e2e-inclusion-proofs-contract: *e2e_test
       - e2e-pending-note-hashes-contract: *e2e_test
       - e2e-ordering: *e2e_test
+      - e2e-outbox: *e2e_test
       - e2e-counter: *e2e_test
       - e2e-private-voting: *e2e_test
       - uniswap-trade-on-l1-from-l2: *e2e_test
@@ -1477,6 +1492,7 @@ workflows:
             - e2e-inclusion-proofs-contract
             - e2e-pending-note-hashes-contract
             - e2e-ordering
+            - e2e-outbox
             - e2e-counter
             - e2e-private-voting
             - uniswap-trade-on-l1-from-l2

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -32,6 +32,7 @@ import {
   Header,
   INITIAL_L2_BLOCK_NUM,
   L1_TO_L2_MSG_TREE_HEIGHT,
+  L2_TO_L1_MESSAGE_LENGTH,
   NOTE_HASH_TREE_HEIGHT,
   NULLIFIER_TREE_HEIGHT,
   NullifierLeafPreimage,
@@ -44,7 +45,8 @@ import { AztecAddress } from '@aztec/foundation/aztec-address';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { AztecKVStore } from '@aztec/kv-store';
 import { AztecLmdbStore } from '@aztec/kv-store/lmdb';
-import { initStoreForRollup } from '@aztec/kv-store/utils';
+import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils';
+import { SHA256, StandardTree } from '@aztec/merkle-tree';
 import { AztecKVTxPool, P2P, createP2PClient } from '@aztec/p2p';
 import {
   GlobalVariableBuilder,
@@ -113,7 +115,7 @@ export class AztecNodeService implements AztecNode {
     const log = createDebugLogger('aztec:node');
     const storeLog = createDebugLogger('aztec:node:lmdb');
     const store = await initStoreForRollup(
-      AztecLmdbStore.open(config.dataDirectory, storeLog),
+      AztecLmdbStore.open(config.dataDirectory, false, storeLog),
       config.l1Contracts.rollupAddress,
       storeLog,
     );
@@ -426,6 +428,47 @@ export class AztecNodeService implements AztecNode {
     return committedDb.getSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, leafIndex);
   }
 
+  /**
+   * Returns the index of a l2ToL1Message in a ephemeral l2 to l1 data tree as well as its sibling path.
+   * @remarks This tree is considered ephemeral because it is created on-demand by: taking all the l2ToL1 messages
+   * in a single block, and then using them to make a variable depth append-only tree with these messages as leaves.
+   * The tree is discarded immediately after calculating what we need from it.
+   * @param blockNumber - The block number at which to get the data.
+   * @param l2ToL1Message - The l2ToL1Message get the index / sibling path for.
+   * @returns A tuple of the index and the sibling path of the L2ToL1Message.
+   */
+  public async getL2ToL1MessageIndexAndSiblingPath(
+    blockNumber: number | 'latest',
+    l2ToL1Message: Fr,
+  ): Promise<[number, SiblingPath<number>]> {
+    const block = await this.blockSource.getBlock(blockNumber === 'latest' ? await this.getBlockNumber() : blockNumber);
+
+    if (block === undefined) {
+      throw new Error('Block is not defined');
+    }
+
+    const l2ToL1Messages = block.body.txEffects.flatMap(txEffect => txEffect.l2ToL1Msgs);
+
+    if (l2ToL1Messages.length !== L2_TO_L1_MESSAGE_LENGTH * block.body.txEffects.length) {
+      throw new Error('L2 to L1 Messages are not padded');
+    }
+
+    const indexOfL2ToL1Message = l2ToL1Messages.findIndex(l2ToL1MessageInBlock =>
+      l2ToL1MessageInBlock.equals(l2ToL1Message),
+    );
+
+    if (indexOfL2ToL1Message === -1) {
+      throw new Error('The L2ToL1Message you are trying to prove inclusion of does not exist');
+    }
+
+    const treeHeight = Math.ceil(Math.log2(l2ToL1Messages.length));
+
+    const tree = new StandardTree(openTmpStore(true), new SHA256(), 'temp_outhash_sibling_path', treeHeight);
+    await tree.appendLeaves(l2ToL1Messages.map(l2ToL1Msg => l2ToL1Msg.toBuffer()));
+
+    return [indexOfL2ToL1Message, await tree.getSiblingPath(BigInt(indexOfL2ToL1Message), true)];
+  }
+
   /**
    * Returns a sibling path for a leaf in the committed blocks tree.
    * @param blockNumber - The block number at which to get the data.

yarn-project/aztec.js/src/index.ts

@@ -115,6 +115,7 @@ export {
   merkleTreeIds,
   mockTx,
   Comparator,
+  SiblingPath,
 } from '@aztec/circuit-types';
 export { NodeInfo } from '@aztec/types/interfaces';
 

yarn-project/aztec/src/cli/cmds/start_archiver.ts

@@ -24,7 +24,7 @@ export const startArchiver = async (options: any, signalHandlers: (() => Promise
 
   const storeLog = createDebugLogger('aztec:archiver:lmdb');
   const store = await initStoreForRollup(
-    AztecLmdbStore.open(archiverConfig.dataDirectory, storeLog),
+    AztecLmdbStore.open(archiverConfig.dataDirectory, false, storeLog),
     archiverConfig.l1Contracts.rollupAddress,
     storeLog,
   );

yarn-project/circuit-types/src/interfaces/aztec-node.ts

@@ -93,6 +93,20 @@ export interface AztecNode {
     leafIndex: bigint,
   ): Promise<SiblingPath<typeof L1_TO_L2_MSG_TREE_HEIGHT>>;
 
+  /**
+   * Returns the index of a l2ToL1Message in a ephemeral l2 to l1 data tree as well as its sibling path.
+   * @remarks This tree is considered ephemeral because it is created on-demand by: taking all the l2ToL1 messages
+   * in a single block, and then using them to make a variable depth append-only tree with these messages as leaves.
+   * The tree is discarded immediately after calculating what we need from it.
+   * @param blockNumber - The block number at which to get the data.
+   * @param l2ToL1Message - The l2ToL1Message get the index / sibling path for.
+   * @returns A tuple of the index and the sibling path of the L2ToL1Message.
+   */
+  getL2ToL1MessageIndexAndSiblingPath(
+    blockNumber: number | 'latest',
+    l2ToL1Message: Fr,
+  ): Promise<[number, SiblingPath<number>]>;
+
   /**
    * Returns a sibling path for a leaf in the committed historic blocks tree.
    * @param blockNumber - The block number at which to get the data.

yarn-project/end-to-end/src/e2e_outbox.test.ts

@@ -0,0 +1,115 @@
+import {
+  AccountWalletWithPrivateKey,
+  AztecNode,
+  BatchCall,
+  DeployL1Contracts,
+  EthAddress,
+  Fr,
+  SiblingPath,
+  sha256,
+} from '@aztec/aztec.js';
+import { SHA256 } from '@aztec/merkle-tree';
+import { TestContract } from '@aztec/noir-contracts.js';
+
+import { beforeEach, describe, expect, it } from '@jest/globals';
+
+import { setup } from './fixtures/utils.js';
+
+// @remark - This does not test the Outbox Contract yet. All this test does is create L2 to L1 messages in a block,
+// verify their existence, and produce a sibling path that is also checked for validity against the circuit produced
+// out_hash in the header.
+describe('E2E Outbox Tests', () => {
+  let teardown: () => void;
+  let aztecNode: AztecNode;
+  const merkleSha256 = new SHA256();
+  let contract: TestContract;
+  let wallets: AccountWalletWithPrivateKey[];
+  let deployL1ContractsValues: DeployL1Contracts;
+
+  beforeEach(async () => {
+    ({ teardown, aztecNode, wallets, deployL1ContractsValues } = await setup(1));
+
+    const receipt = await TestContract.deploy(wallets[0]).send({ contractAddressSalt: Fr.ZERO }).wait();
+    contract = receipt.contract;
+  }, 100_000);
+
+  afterAll(() => teardown());
+
+  it('Inserts a new transaction with two out messages, and verifies sibling paths of both the new messages', async () => {
+    const [[recipient1, content1], [recipient2, content2]] = [
+      [EthAddress.random(), Fr.random()],
+      [EthAddress.random(), Fr.random()],
+    ];
+
+    // We can't put any more l2 to L1 messages here There are a max of 2 L2 to L1 messages per transaction
+    const call = new BatchCall(wallets[0], [
+      contract.methods.create_l2_to_l1_message_arbitrary_recipient_private(content1, recipient1).request(),
+      contract.methods.create_l2_to_l1_message_arbitrary_recipient_private(content2, recipient2).request(),
+    ]);
+
+    // TODO (#5104): When able to guarantee multiple txs in a single block, make this populate a full tree. Right now we are
+    // unable to do this because in CI, for some reason, the tx's are handled in different blocks, so it is impossible
+    // to make a full tree of L2 -> L1 messages as we are only able to set one tx's worth of L1 -> L2 messages in a block (2 messages out of 4)
+    const txReceipt = await call.send().wait();
+
+    const block = await aztecNode.getBlock(txReceipt.blockNumber!);
+
+    const l2ToL1Messages = block?.body.txEffects.flatMap(txEffect => txEffect.l2ToL1Msgs);
+
+    expect(l2ToL1Messages?.map(l2ToL1Message => l2ToL1Message.toString())).toStrictEqual(
+      [makeL2ToL1Message(recipient2, content2), makeL2ToL1Message(recipient1, content1), Fr.ZERO, Fr.ZERO].map(
+        expectedL2ToL1Message => expectedL2ToL1Message.toString(),
+      ),
+    );
+
+    // For each individual message, we are using our node API to grab the index and sibling path. We expect
+    // the index to match the order of the block we obtained earlier. We also then use this sibling path to hash up to the root,
+    // verifying that the expected root obtained through the message and the sibling path match the actual root
+    // that was returned by the circuits in the header as out_hash.
+    const [index, siblingPath] = await aztecNode.getL2ToL1MessageIndexAndSiblingPath(
+      txReceipt.blockNumber!,
+      l2ToL1Messages![0],
+    );
+    expect(siblingPath.pathSize).toBe(2);
+    expect(index).toBe(0);
+    const expectedRoot = calculateExpectedRoot(l2ToL1Messages![0], siblingPath as SiblingPath<2>, index);
+    expect(expectedRoot.toString('hex')).toEqual(block?.header.contentCommitment.outHash.toString('hex'));
+
+    const [index2, siblingPath2] = await aztecNode.getL2ToL1MessageIndexAndSiblingPath(
+      txReceipt.blockNumber!,
+      l2ToL1Messages![1],
+    );
+    expect(siblingPath2.pathSize).toBe(2);
+    expect(index2).toBe(1);
+    const expectedRoot2 = calculateExpectedRoot(l2ToL1Messages![1], siblingPath2 as SiblingPath<2>, index2);
+    expect(expectedRoot2.toString('hex')).toEqual(block?.header.contentCommitment.outHash.toString('hex'));
+  }, 360_000);
+
+  function calculateExpectedRoot(l2ToL1Message: Fr, siblingPath: SiblingPath<2>, index: number): Buffer {
+    const firstLayerInput: [Buffer, Buffer] =
+      index & 0x1
+        ? [siblingPath.toBufferArray()[0], l2ToL1Message.toBuffer()]
+        : [l2ToL1Message.toBuffer(), siblingPath.toBufferArray()[0]];
+    const firstLayer = merkleSha256.hash(...firstLayerInput);
+    index /= 2;
+    const secondLayerInput: [Buffer, Buffer] =
+      index & 0x1 ? [siblingPath.toBufferArray()[1], firstLayer] : [firstLayer, siblingPath.toBufferArray()[1]];
+    return merkleSha256.hash(...secondLayerInput);
+  }
+
+  function makeL2ToL1Message(recipient: EthAddress, content: Fr = Fr.ZERO): Fr {
+    const leaf = Fr.fromBufferReduce(
+      sha256(
+        Buffer.concat([
+          contract.address.toBuffer(),
+          new Fr(1).toBuffer(), // aztec version
+          recipient.toBuffer32(),
+          new Fr(deployL1ContractsValues.publicClient.chain.id).toBuffer(), // chain id
+          content.toBuffer(),
+        ]),
+      ),
+    );
+
+    return leaf;
+  }
+});

yarn-project/kv-store/src/lmdb/store.ts

@@ -45,12 +45,20 @@ export class AztecLmdbStore implements AztecKVStore {
    * different rollup instances.
    *
    * @param path - A path on the disk to store the database. Optional
+   * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
    * @param log - A logger to use. Optional
    * @returns The store
    */
-  static open(path?: string, log = createDebugLogger('aztec:kv-store:lmdb')): AztecLmdbStore {
+  static open(
+    path?: string,
+    ephemeral: boolean = false,
+    log = createDebugLogger('aztec:kv-store:lmdb'),
+  ): AztecLmdbStore {
     log.info(`Opening LMDB database at ${path || 'temporary location'}`);
-    const rootDb = open({ path });
+    const rootDb = open({
+      path,
+      noSync: ephemeral,
+    });
     return new AztecLmdbStore(rootDb);
   }
 

yarn-project/kv-store/src/utils.ts

@@ -34,8 +34,9 @@ export async function initStoreForRollup<T extends AztecKVStore>(
 
 /**
  * Opens a temporary store for testing purposes.
+ * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
  * @returns A new store
  */
-export function openTmpStore(): AztecKVStore {
-  return AztecLmdbStore.open();
+export function openTmpStore(ephemeral: boolean = false): AztecKVStore {
+  return AztecLmdbStore.open(undefined, ephemeral);
 }

yarn-project/merkle-tree/src/index.ts

@@ -3,6 +3,7 @@ export * from './interfaces/indexed_tree.js';
 export * from './interfaces/merkle_tree.js';
 export * from './interfaces/update_only_tree.js';
 export * from './pedersen.js';
+export * from './sha_256.js';
 export * from './sparse_tree/sparse_tree.js';
 export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tree.js';
 export { StandardIndexedTreeWithAppend } from './standard_indexed_tree/test/standard_indexed_tree_with_append.js';

yarn-project/merkle-tree/src/sha_256.ts

@@ -0,0 +1,25 @@
+import { sha256 } from '@aztec/foundation/crypto';
+import { Hasher } from '@aztec/types/interfaces';
+
+/**
+ * A helper class encapsulating SHA256 hash functionality.
+ * @deprecated Don't call SHA256 directly in production code. Instead, create suitably-named functions for specific
+ * purposes.
+ */
+export class SHA256 implements Hasher {
+  /*
+   * @deprecated Don't call SHA256 directly in production code. Instead, create suitably-named functions for specific
+   * purposes.
+   */
+  public hash(lhs: Uint8Array, rhs: Uint8Array): Buffer {
+    return sha256(Buffer.concat([Buffer.from(lhs), Buffer.from(rhs)]));
+  }
+
+  /*
+   * @deprecated Don't call SHA256 directly in production code. Instead, create suitably-named functions for specific
+   * purposes.
+   */
+  public hashInputs(inputs: Buffer[]): Buffer {
+    return sha256(Buffer.concat(inputs));
+  }
+}