feat: Get logs by tags

yarn-project/archiver/src/archiver/archiver.ts

@@ -1,4 +1,5 @@
 import {
+  type EncryptedL2NoteLog,
   type FromLogType,
   type GetUnencryptedLogsResponse,
   type InboxLeaf,
@@ -627,6 +628,10 @@ export class Archiver implements ArchiveSource {
     return this.store.getLogs(from, limit, logType);
   }
 
+  getLogsByTags(tags: Fr[]): Promise<EncryptedL2NoteLog[][]> {
+    return this.store.getLogsByTags(tags);
+  }
+
   /**
    * Gets unencrypted logs based on the provided filter.
    * @param filter - The filter to apply to the logs.
@@ -924,6 +929,9 @@ class ArchiverStoreHelper
   ): Promise<L2BlockL2Logs<FromLogType<TLogType>>[]> {
     return this.store.getLogs(from, limit, logType);
   }
+  getLogsByTags(tags: Fr[]): Promise<EncryptedL2NoteLog[][]> {
+    return this.store.getLogsByTags(tags);
+  }
   getUnencryptedLogs(filter: LogFilter): Promise<GetUnencryptedLogsResponse> {
     return this.store.getUnencryptedLogs(filter);
   }

yarn-project/archiver/src/archiver/archiver_store.ts

@@ -1,4 +1,5 @@
 import {
+  type EncryptedL2NoteLog,
   type FromLogType,
   type GetUnencryptedLogsResponse,
   type InboxLeaf,
@@ -136,6 +137,8 @@ export interface ArchiverDataStore {
     logType: TLogType,
   ): Promise<L2BlockL2Logs<FromLogType<TLogType>>[]>;
 
+  getLogsByTags(tags: Fr[]): Promise<EncryptedL2NoteLog[][]>;
+
   /**
    * Gets unencrypted logs based on the provided filter.
    * @param filter - The filter to apply to the logs.

yarn-project/archiver/src/archiver/archiver_store_test_suite.ts

@@ -354,6 +354,79 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch
       });
     });
 
+    describe('getLogsByTags', () => {
+      const txsPerBlock = 4;
+      const numPrivateFunctionCalls = 3;
+      const numNoteEncryptedLogs = 2;
+      const numBlocks = 10;
+      let blocks: L1Published<L2Block>[];
+      let tags: { [i: number]: { [j: number]: Fr[] } } = {};
+
+      beforeEach(async () => {
+        blocks = times(numBlocks, (index: number) => ({
+          data: L2Block.random(index + 1, txsPerBlock, numPrivateFunctionCalls, 2, numNoteEncryptedLogs, 2),
+          l1: { blockNumber: BigInt(index), blockHash: `0x${index}`, timestamp: BigInt(index) },
+        }));
+
+        await store.addBlocks(blocks);
+        await store.addLogs(blocks.map(b => b.data));
+
+        tags = {};
+        blocks.forEach((b, blockIndex) => {
+          if (!tags[blockIndex]) {
+            tags[blockIndex] = {};
+          }
+          b.data.body.noteEncryptedLogs.txLogs.forEach((txLogs, txIndex) => {
+            if (!tags[blockIndex][txIndex]) {
+              tags[blockIndex][txIndex] = [];
+            }
+            tags[blockIndex][txIndex].push(...txLogs.unrollLogs().map(log => new Fr(log.data.subarray(0, 32))));
+          });
+        });
+      });
+
+      it('is possible to batch request all logs of a tx via tags', async () => {
+        // get random tx
+        const targetBlockIndex = randomInt(numBlocks);
+        const targetTxIndex = randomInt(txsPerBlock);
+
+        const logs = await store.getLogsByTags(tags[targetBlockIndex][targetTxIndex]);
+
+        const expectedNumLogs = numPrivateFunctionCalls * numNoteEncryptedLogs;
+        expect(logs.length).toEqual(expectedNumLogs);
+
+        logs.forEach(log => expect(log).toHaveLength(1));
+      });
+
+      it('is possible to batch request all logs of different blocks via tags', async () => {
+        // get random tx
+        const targetBlockIndex = randomInt(numBlocks);
+        const targetTxIndex = randomInt(txsPerBlock);
+
+        const logs = await store.getLogsByTags([...tags[0][0], ...tags[targetBlockIndex][targetTxIndex]]);
+
+        const expectedNumLogs = 2 * numPrivateFunctionCalls * numNoteEncryptedLogs;
+        expect(logs.length).toEqual(expectedNumLogs);
+
+        logs.forEach(log => expect(log).toHaveLength(1));
+      });
+
+      it('is possible to request logs for non-existing tags and determine their position', async () => {
+        // get random tx
+        const targetBlockIndex = randomInt(numBlocks);
+        const targetTxIndex = randomInt(txsPerBlock);
+
+        const logs = await store.getLogsByTags([Fr.random(), ...tags[targetBlockIndex][targetTxIndex].slice(1)]);
+
+        const expectedNumLogs = numPrivateFunctionCalls * numNoteEncryptedLogs;
+        expect(logs.length).toEqual(expectedNumLogs);
+
+        expect(logs[0]).toHaveLength(0);
+
+        logs.slice(1).forEach(log => expect(log).toHaveLength(1));
+      });
+    });
+
     describe('getUnencryptedLogs', () => {
       const txsPerBlock = 4;
       const numPublicFunctionCalls = 3;

yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts

@@ -1,4 +1,5 @@
 import {
+  type EncryptedL2NoteLog,
   type FromLogType,
   type GetUnencryptedLogsResponse,
   type InboxLeaf,
@@ -239,6 +240,14 @@ export class KVArchiverDataStore implements ArchiverDataStore {
     }
   }
 
+  getLogsByTags(tags: Fr[]): Promise<EncryptedL2NoteLog[][]> {
+    try {
+      return this.#logStore.getLogsByTags(tags);
+    } catch (err) {
+      return Promise.reject(err);
+    }
+  }
+
   /**
    * Gets unencrypted logs based on the provided filter.
    * @param filter - The filter to apply to the logs.

yarn-project/archiver/src/archiver/kv_archiver_store/log_store.ts

@@ -1,5 +1,6 @@
 import {
   EncryptedL2BlockL2Logs,
+  EncryptedL2NoteLog,
   EncryptedNoteL2BlockL2Logs,
   ExtendedUnencryptedL2Log,
   type FromLogType,
@@ -12,26 +13,33 @@ import {
   UnencryptedL2BlockL2Logs,
   type UnencryptedL2Log,
 } from '@aztec/circuit-types';
+import { Fr } from '@aztec/circuits.js';
 import { INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js/constants';
 import { createDebugLogger } from '@aztec/foundation/log';
-import { type AztecKVStore, type AztecMap } from '@aztec/kv-store';
+import { type AztecKVStore, type AztecMap, type AztecMultiMap } from '@aztec/kv-store';
 
 import { type BlockStore } from './block_store.js';
 
 /**
  * A store for logs
  */
 export class LogStore {
-  #noteEncryptedLogs: AztecMap<number, Buffer>;
-  #encryptedLogs: AztecMap<number, Buffer>;
-  #unencryptedLogs: AztecMap<number, Buffer>;
+  #noteEncryptedLogsByBlock: AztecMap<number, Buffer>;
+  #noteEncryptedLogsByHash: AztecMap<string, Buffer>;
+  #noteEncryptedLogHashesByTag: AztecMultiMap<string, string>;
+  #noteEncryptedLogTagsByBlock: AztecMultiMap<number, string>;
+  #encryptedLogsByBlock: AztecMap<number, Buffer>;
+  #unencryptedLogsByBlock: AztecMap<number, Buffer>;
   #logsMaxPageSize: number;
   #log = createDebugLogger('aztec:archiver:log_store');
 
   constructor(private db: AztecKVStore, private blockStore: BlockStore, logsMaxPageSize: number = 1000) {
-    this.#noteEncryptedLogs = db.openMap('archiver_note_encrypted_logs');
-    this.#encryptedLogs = db.openMap('archiver_encrypted_logs');
-    this.#unencryptedLogs = db.openMap('archiver_unencrypted_logs');
+    this.#noteEncryptedLogsByBlock = db.openMap('archiver_note_encrypted_logs');
+    this.#noteEncryptedLogsByHash = db.openMap('archiver_note_encrypted_logs_by_hash');
+    this.#noteEncryptedLogHashesByTag = db.openMultiMap('archiver_tagged_note_encrypted_logs');
+    this.#noteEncryptedLogTagsByBlock = db.openMultiMap('archiver_note_encrypted_log_tags_by_block');
+    this.#encryptedLogsByBlock = db.openMap('archiver_encrypted_logs');
+    this.#unencryptedLogsByBlock = db.openMap('archiver_unencrypted_logs');
 
     this.#logsMaxPageSize = logsMaxPageSize;
   }
@@ -44,21 +52,50 @@ export class LogStore {
   addLogs(blocks: L2Block[]): Promise<boolean> {
     return this.db.transaction(() => {
       blocks.forEach(block => {
-        void this.#noteEncryptedLogs.set(block.number, block.body.noteEncryptedLogs.toBuffer());
-        void this.#encryptedLogs.set(block.number, block.body.encryptedLogs.toBuffer());
-        void this.#unencryptedLogs.set(block.number, block.body.unencryptedLogs.toBuffer());
+        void this.#noteEncryptedLogsByBlock.set(block.number, block.body.noteEncryptedLogs.toBuffer());
+        block.body.noteEncryptedLogs.txLogs.forEach(txLogs => {
+          const noteLogs = txLogs.unrollLogs();
+          noteLogs.forEach(noteLog => {
+            const tag = new Fr(noteLog.data.subarray(0, 32));
+            const hexHash = noteLog.hash().toString('hex');
+            // AztecMultiMap doesn't handle storing buffers well. Using the body 'ordered-binary' encoding returns an error
+            // when trying to decode buffers (The number <> cannot be converted to a BigInt because it is not an integer)
+            // This unfortunately leads to some duplication, but using the hash of the note log should be fine
+            // even if we had duplicated tags.
+            void this.#noteEncryptedLogHashesByTag.set(tag.toString(), hexHash);
+            void this.#noteEncryptedLogsByHash.set(hexHash, noteLog.toBuffer());
+            void this.#noteEncryptedLogTagsByBlock.set(block.number, tag.toString());
+          });
+        });
+        void this.#encryptedLogsByBlock.set(block.number, block.body.encryptedLogs.toBuffer());
+        void this.#unencryptedLogsByBlock.set(block.number, block.body.unencryptedLogs.toBuffer());
       });
 
       return true;
     });
   }
 
-  deleteLogs(blocks: L2Block[]): Promise<boolean> {
+  async deleteLogs(blocks: L2Block[]): Promise<boolean> {
+    const noteTagsToDelete = await this.db.transaction(() => {
+      return blocks.flatMap(block => Array.from(this.#noteEncryptedLogTagsByBlock.getValues(block.number)));
+    });
+    const noteLogHashesToDelete = await this.db.transaction(() => {
+      return noteTagsToDelete.flatMap(tag => Array.from(this.#noteEncryptedLogHashesByTag.getValues(tag)));
+    });
     return this.db.transaction(() => {
       blocks.forEach(block => {
-        void this.#noteEncryptedLogs.delete(block.number);
-        void this.#encryptedLogs.delete(block.number);
-        void this.#unencryptedLogs.delete(block.number);
+        void this.#noteEncryptedLogsByBlock.delete(block.number);
+        void this.#encryptedLogsByBlock.delete(block.number);
+        void this.#unencryptedLogsByBlock.delete(block.number);
+        void this.#noteEncryptedLogTagsByBlock.delete(block.number);
+      });
+
+      noteTagsToDelete.forEach(tag => {
+        void this.#noteEncryptedLogHashesByTag.delete(tag.toString());
+      });
+
+      noteLogHashesToDelete.forEach(hash => {
+        void this.#noteEncryptedLogsByHash.delete(hash);
       });
 
       return true;
@@ -80,12 +117,12 @@ export class LogStore {
     const logMap = (() => {
       switch (logType) {
         case LogType.ENCRYPTED:
-          return this.#encryptedLogs;
+          return this.#encryptedLogsByBlock;
         case LogType.NOTEENCRYPTED:
-          return this.#noteEncryptedLogs;
+          return this.#noteEncryptedLogsByBlock;
         case LogType.UNENCRYPTED:
         default:
-          return this.#unencryptedLogs;
+          return this.#unencryptedLogsByBlock;
       }
     })();
     const logTypeMap = (() => {
@@ -105,6 +142,21 @@ export class LogStore {
     }
   }
 
+  getLogsByTags(tags: Fr[]): Promise<EncryptedL2NoteLog[][]> {
+    return this.db.transaction(() => {
+      return tags.map(tag => {
+        const logHashes = Array.from(this.#noteEncryptedLogHashesByTag.getValues(tag.toString()));
+        return (
+          logHashes
+            .map(hash => this.#noteEncryptedLogsByHash.get(hash))
+            // filter out undefined values, since we should never store the hashes of non-existing logs (the addLogs transaction ensures this)
+            .filter(noteLogBuffer => noteLogBuffer)
+            .map(noteLogBuffer => EncryptedL2NoteLog.fromBuffer(noteLogBuffer!))
+        );
+      });
+    });
+  }
+
   /**
    * Gets unencrypted logs based on the provided filter.
    * @param filter - The filter to apply to the logs.
@@ -154,7 +206,7 @@ export class LogStore {
     const logs: ExtendedUnencryptedL2Log[] = [];
 
     let maxLogsHit = false;
-    loopOverBlocks: for (const [blockNumber, logBuffer] of this.#unencryptedLogs.entries({ start, end })) {
+    loopOverBlocks: for (const [blockNumber, logBuffer] of this.#unencryptedLogsByBlock.entries({ start, end })) {
       const unencryptedLogsInBlock = UnencryptedL2BlockL2Logs.fromBuffer(logBuffer);
       for (let txIndex = filter.afterLog?.txIndex ?? 0; txIndex < unencryptedLogsInBlock.txLogs.length; txIndex++) {
         const txLogs = unencryptedLogsInBlock.txLogs[txIndex].unrollLogs();
@@ -199,12 +251,12 @@ export class LogStore {
     const logMap = (() => {
       switch (logType) {
         case LogType.ENCRYPTED:
-          return this.#encryptedLogs;
+          return this.#encryptedLogsByBlock;
         case LogType.NOTEENCRYPTED:
-          return this.#noteEncryptedLogs;
+          return this.#noteEncryptedLogsByBlock;
         case LogType.UNENCRYPTED:
         default:
-          return this.#unencryptedLogs;
+          return this.#unencryptedLogsByBlock;
       }
     })();
     const logTypeMap = (() => {

yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts

@@ -1,5 +1,6 @@
 import {
   type EncryptedL2BlockL2Logs,
+  type EncryptedL2NoteLog,
   type EncryptedNoteL2BlockL2Logs,
   ExtendedUnencryptedL2Log,
   type FromLogType,
@@ -49,6 +50,10 @@ export class MemoryArchiverStore implements ArchiverDataStore {
 
   private noteEncryptedLogsPerBlock: Map<number, EncryptedNoteL2BlockL2Logs> = new Map();
 
+  private taggedNoteEncryptedLogs: Map<string, EncryptedL2NoteLog[]> = new Map();
+
+  private noteEncryptedLogTagsPerBlock: Map<number, Fr[]> = new Map();
+
   private encryptedLogsPerBlock: Map<number, EncryptedL2BlockL2Logs> = new Map();
 
   private unencryptedLogsPerBlock: Map<number, UnencryptedL2BlockL2Logs> = new Map();
@@ -206,17 +211,35 @@ export class MemoryArchiverStore implements ArchiverDataStore {
   addLogs(blocks: L2Block[]): Promise<boolean> {
     blocks.forEach(block => {
       this.noteEncryptedLogsPerBlock.set(block.number, block.body.noteEncryptedLogs);
+      block.body.noteEncryptedLogs.txLogs.forEach(txLogs => {
+        const noteLogs = txLogs.unrollLogs();
+        noteLogs.forEach(noteLog => {
+          const tag = new Fr(noteLog.data.subarray(0, 32));
+          const currentNoteLogs = this.taggedNoteEncryptedLogs.get(tag.toString()) || [];
+          this.taggedNoteEncryptedLogs.set(tag.toString(), [...currentNoteLogs, noteLog]);
+          const currentTagsInBlock = this.noteEncryptedLogTagsPerBlock.get(block.number) || [];
+          this.noteEncryptedLogTagsPerBlock.set(block.number, [...currentTagsInBlock, tag]);
+        });
+      });
       this.encryptedLogsPerBlock.set(block.number, block.body.encryptedLogs);
       this.unencryptedLogsPerBlock.set(block.number, block.body.unencryptedLogs);
     });
     return Promise.resolve(true);
   }
 
   deleteLogs(blocks: L2Block[]): Promise<boolean> {
+    const noteTagsToDelete = blocks.flatMap(block => this.noteEncryptedLogTagsPerBlock.get(block.number));
+    noteTagsToDelete
+      .filter(tag => tag)
+      .forEach(tag => {
+        this.taggedNoteEncryptedLogs.delete(tag!.toString());
+      });
+
     blocks.forEach(block => {
       this.encryptedLogsPerBlock.delete(block.number);
       this.noteEncryptedLogsPerBlock.delete(block.number);
       this.unencryptedLogsPerBlock.delete(block.number);
+      this.noteEncryptedLogTagsPerBlock.delete(block.number);
     });
 
     return Promise.resolve(true);
@@ -380,6 +403,14 @@ export class MemoryArchiverStore implements ArchiverDataStore {
     return Promise.resolve(l);
   }
 
+  getLogsByTags(tags: Fr[]): Promise<EncryptedL2NoteLog[][]> {
+    const noteLogs = tags.map(tag => {
+      const noteLogs = this.taggedNoteEncryptedLogs.get(tag.toString());
+      return noteLogs ? noteLogs : [];
+    });
+    return Promise.resolve(noteLogs);
+  }
+
   /**
    * Gets unencrypted logs based on the provided filter.
    * @param filter - The filter to apply to the logs.

yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts

@@ -1,5 +1,6 @@
 import {
   type AztecNode,
+  EncryptedL2NoteLog,
   EncryptedNoteL2BlockL2Logs,
   EpochProofQuote,
   ExtendedUnencryptedL2Log,
@@ -49,6 +50,7 @@ export function createAztecNodeRpcServer(node: AztecNode) {
     },
     {
       EncryptedNoteL2BlockL2Logs,
+      EncryptedL2NoteLog,
       NoteSelector,
       NullifierMembershipWitness,
       PublicSimulationOutput,