diff --git a/yarn-project/pxe/src/contract_function_simulator/execution_tagging_index_cache.ts b/yarn-project/pxe/src/contract_function_simulator/execution_tagging_index_cache.ts index 27612bf8ceaf..37ffc83016d9 100644 --- a/yarn-project/pxe/src/contract_function_simulator/execution_tagging_index_cache.ts +++ b/yarn-project/pxe/src/contract_function_simulator/execution_tagging_index_cache.ts @@ -1,37 +1,32 @@ -import { ExtendedDirectionalAppTaggingSecret, type TaggingIndexRange } from '@aztec/stdlib/logs'; +import { ExtendedDirectionalAppTaggingSecret, type PreTag } from '@aztec/stdlib/logs'; /** - * A map that stores the tagging index range for a given extended directional app tagging secret. + * A map that stores the tagging index for a given extended directional app tagging secret. * Note: The directional app tagging secret is unique for a (sender, recipient, contract) tuple while the direction * of sender -> recipient matters. */ export class ExecutionTaggingIndexCache { - private taggingIndexMap: Map = new Map(); + private taggingIndexMap: Map = new Map(); public getLastUsedIndex(secret: ExtendedDirectionalAppTaggingSecret): number | undefined { - return this.taggingIndexMap.get(secret.toString())?.highestIndex; + return this.taggingIndexMap.get(secret.toString()); } public setLastUsedIndex(secret: ExtendedDirectionalAppTaggingSecret, index: number) { const currentValue = this.taggingIndexMap.get(secret.toString()); - if (currentValue !== undefined && currentValue.highestIndex !== index - 1) { - throw new Error(`Invalid tagging index update. Current value: ${currentValue.highestIndex}, new value: ${index}`); - } - if (currentValue !== undefined) { - currentValue.highestIndex = index; - } else { - this.taggingIndexMap.set(secret.toString(), { lowestIndex: index, highestIndex: index }); + if (currentValue !== undefined && currentValue !== index - 1) { + throw new Error(`Invalid tagging index update. Current value: ${currentValue}, new value: ${index}`); } + this.taggingIndexMap.set(secret.toString(), index); } /** - * Returns the tagging index ranges that were used in this execution (and that need to be stored in the db). + * Returns the pre-tags that were used in this execution (and that need to be stored in the db). */ - public getUsedTaggingIndexRanges(): TaggingIndexRange[] { - return Array.from(this.taggingIndexMap.entries()).map(([secret, { lowestIndex, highestIndex }]) => ({ + public getUsedPreTags(): PreTag[] { + return Array.from(this.taggingIndexMap.entries()).map(([secret, index]) => ({ extendedSecret: ExtendedDirectionalAppTaggingSecret.fromString(secret), - lowestIndex, - highestIndex, + index, })); } } diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.ts index e9eeabca6d20..e25c99930f3f 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.ts @@ -81,7 +81,7 @@ export async function executePrivateFunction( const newNotes = privateExecutionOracle.getNewNotes(); const noteHashNullifierCounterMap = privateExecutionOracle.getNoteHashNullifierCounterMap(); const offchainEffects = privateExecutionOracle.getOffchainEffects(); - const taggingIndexRanges = privateExecutionOracle.getUsedTaggingIndexRanges(); + const preTags = privateExecutionOracle.getUsedPreTags(); const nestedExecutionResults = privateExecutionOracle.getNestedExecutionResults(); let timerSubtractionList = nestedExecutionResults; @@ -104,7 +104,7 @@ export async function executePrivateFunction( noteHashNullifierCounterMap, rawReturnValues, offchainEffects, - taggingIndexRanges, + preTags, nestedExecutionResults, contractClassLogs, { diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts index 348dbc7ab593..a1b2ada7881e 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts @@ -14,7 +14,7 @@ import { import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { siloNullifier } from '@aztec/stdlib/hash'; import { PrivateContextInputs } from '@aztec/stdlib/kernel'; -import { type ContractClassLog, ExtendedDirectionalAppTaggingSecret, type TaggingIndexRange } from '@aztec/stdlib/logs'; +import { type ContractClassLog, ExtendedDirectionalAppTaggingSecret, type PreTag } from '@aztec/stdlib/logs'; import { Tag } from '@aztec/stdlib/logs'; import { Note, type NoteStatus } from '@aztec/stdlib/note'; import { @@ -166,10 +166,10 @@ export class PrivateExecutionOracle extends UtilityExecutionOracle implements IP } /** - * Returns the tagging index ranges that were used in this execution (and that need to be stored in the db). + * Returns the pre-tags that were used in this execution (and that need to be stored in the db). */ - public getUsedTaggingIndexRanges(): TaggingIndexRange[] { - return this.taggingIndexCache.getUsedTaggingIndexRanges(); + public getUsedPreTags(): PreTag[] { + return this.taggingIndexCache.getUsedPreTags(); } /** diff --git a/yarn-project/pxe/src/pxe.ts b/yarn-project/pxe/src/pxe.ts index 5ff941da05db..9b7e5cc3ed98 100644 --- a/yarn-project/pxe/src/pxe.ts +++ b/yarn-project/pxe/src/pxe.ts @@ -764,17 +764,17 @@ export class PXE { // transaction before this one is included in a block from this PXE, and that transaction contains a log with // a tag derived from the same secret, we would reuse the tag and the transactions would be linked. Hence // storing the tags here prevents linkage of txs sent from the same PXE. - const taggingIndexRangesUsedInTheTx = privateExecutionResult.entrypoint.taggingIndexRanges; - if (taggingIndexRangesUsedInTheTx.length > 0) { + const preTagsUsedInTheTx = privateExecutionResult.entrypoint.preTags; + if (preTagsUsedInTheTx.length > 0) { // TODO(benesjan): The following is an expensive operation. Figure out a way to avoid it. const txHash = (await txProvingResult.toTx()).txHash; - await this.senderTaggingStore.storePendingIndexes(taggingIndexRangesUsedInTheTx, txHash, jobId); - this.log.debug(`Stored used tagging index ranges as sender for the tx`, { - taggingIndexRangesUsedInTheTx, + await this.senderTaggingStore.storePendingIndexes(preTagsUsedInTheTx, txHash, jobId); + this.log.debug(`Stored used pre-tags as sender for the tx`, { + preTagsUsedInTheTx, }); } else { - this.log.debug(`No tagging index ranges used in the tx`); + this.log.debug(`No pre-tags used in the tx`); } return txProvingResult; diff --git a/yarn-project/pxe/src/storage/metadata.ts b/yarn-project/pxe/src/storage/metadata.ts index 826f90735b91..cb1dee391377 100644 --- a/yarn-project/pxe/src/storage/metadata.ts +++ b/yarn-project/pxe/src/storage/metadata.ts @@ -1 +1 @@ -export const PXE_DATA_SCHEMA_VERSION = 4; +export const PXE_DATA_SCHEMA_VERSION = 3; diff --git a/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts b/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts index b2800582f02d..986f1daef6fc 100644 --- a/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts +++ b/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts @@ -1,19 +1,11 @@ -import { Fr } from '@aztec/foundation/curves/bn254'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; -import { RevertCode } from '@aztec/stdlib/avm'; -import type { ExtendedDirectionalAppTaggingSecret, TaggingIndexRange } from '@aztec/stdlib/logs'; -import { PrivateLog, SiloedTag } from '@aztec/stdlib/logs'; +import type { ExtendedDirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs'; import { randomExtendedDirectionalAppTaggingSecret } from '@aztec/stdlib/testing'; -import { TxEffect, TxHash } from '@aztec/stdlib/tx'; +import { TxHash } from '@aztec/stdlib/tx'; import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/constants.js'; import { SenderTaggingStore } from './sender_tagging_store.js'; -/** Helper to create a single-index range (lowestIndex === highestIndex). */ -function range(secret: ExtendedDirectionalAppTaggingSecret, lowest: number, highest?: number): TaggingIndexRange { - return { extendedSecret: secret, lowestIndex: lowest, highestIndex: highest ?? lowest }; -} - describe('SenderTaggingStore', () => { let taggingStore: SenderTaggingStore; let secret1: ExtendedDirectionalAppTaggingSecret; @@ -26,20 +18,25 @@ describe('SenderTaggingStore', () => { }); describe('storePendingIndexes', () => { - it('stores a single pending index range', async () => { + it('stores a single pending index', async () => { const txHash = TxHash.random(); + const preTag: PreTag = { extendedSecret: secret1, index: 5 }; - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); + await taggingStore.storePendingIndexes([preTag], txHash, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); expect(txHashes).toHaveLength(1); expect(txHashes[0]).toEqual(txHash); }); - it('stores multiple pending index ranges for different secrets', async () => { + it('stores multiple pending indexes for different secrets', async () => { const txHash = TxHash.random(); + const preTags: PreTag[] = [ + { extendedSecret: secret1, index: 3 }, + { extendedSecret: secret2, index: 7 }, + ]; - await taggingStore.storePendingIndexes([range(secret1, 3), range(secret2, 7)], txHash, 'test'); + await taggingStore.storePendingIndexes(preTags, txHash, 'test'); const txHashes1 = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); expect(txHashes1).toHaveLength(1); @@ -50,12 +47,12 @@ describe('SenderTaggingStore', () => { expect(txHashes2[0]).toEqual(txHash); }); - it('stores multiple pending index ranges for the same secret from different txs', async () => { + it('stores multiple pending indexes for the same secret from different txs', async () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); expect(txHashes).toHaveLength(2); @@ -63,71 +60,68 @@ describe('SenderTaggingStore', () => { expect(txHashes).toContainEqual(txHash2); }); - it('ignores duplicate range + txHash combination', async () => { + it('ignores duplicate preTag + txHash combination', async () => { const txHash = TxHash.random(); + const preTag: PreTag = { extendedSecret: secret1, index: 5 }; - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); + await taggingStore.storePendingIndexes([preTag], txHash, 'test'); + await taggingStore.storePendingIndexes([preTag], txHash, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); expect(txHashes).toHaveLength(1); expect(txHashes[0]).toEqual(txHash); }); - it('stores a range spanning multiple indexes', async () => { + it('throws when storing duplicate secrets in the same call', async () => { const txHash = TxHash.random(); + const preTags: PreTag[] = [ + { extendedSecret: secret1, index: 3 }, + { extendedSecret: secret1, index: 7 }, + ]; - await taggingStore.storePendingIndexes([range(secret1, 3, 7)], txHash, 'test'); - - // By design the txs are filtered based on the highestIndex (7) in getTxHashesOfPendingIndexes so we shouldn't - // receive the tx only in the second query. - const txHashesNotContainingHighest = await taggingStore.getTxHashesOfPendingIndexes(secret1, 3, 4, 'test'); - expect(txHashesNotContainingHighest).toHaveLength(0); - - const txHashesContainingHighest = await taggingStore.getTxHashesOfPendingIndexes(secret1, 7, 8, 'test'); - expect(txHashesContainingHighest).toHaveLength(1); - expect(txHashesContainingHighest[0]).toEqual(txHash); - - expect(await taggingStore.getLastUsedIndex(secret1, 'test')).toBe(7); + await expect(taggingStore.storePendingIndexes(preTags, txHash, 'test')).rejects.toThrow( + 'Duplicate secrets found when storing pending indexes', + ); }); - it('throws when storing a different range for an existing secret + txHash pair', async () => { + it('throws when storing a different index for an existing secret + txHash pair', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); + // First store an index + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash, 'test'); - // Storing a different range for the same secret + txHash should throw - await expect(taggingStore.storePendingIndexes([range(secret1, 7)], txHash, 'test')).rejects.toThrow( - /Conflicting range/, - ); + // Try to store a different index for the same secret + txHash pair + await expect( + taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash, 'test'), + ).rejects.toThrow(/Cannot store index 7.*a different index 5 already exists/); }); - it('throws when storing a pending index range lower than the last finalized index', async () => { + it('throws when storing a pending index lower than the last finalized index', async () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); // First store and finalize an index - await taggingStore.storePendingIndexes([range(secret1, 10)], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 10 }], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Try to store a pending index lower than the finalized index - await expect(taggingStore.storePendingIndexes([range(secret1, 5)], txHash2, 'test')).rejects.toThrow( - /lowestIndex is lower than or equal to the last finalized index 10/, - ); + await expect( + taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash2, 'test'), + ).rejects.toThrow(/Cannot store pending index 5.*lower than or equal to the last finalized index 10/); }); - it('throws when storing a pending index range equal to the last finalized index', async () => { + it('throws when storing a pending index equal to the last finalized index', async () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); // First store and finalize an index - await taggingStore.storePendingIndexes([range(secret1, 10)], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 10 }], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Try to store a pending index equal to the finalized index - await expect(taggingStore.storePendingIndexes([range(secret1, 10)], txHash2, 'test')).rejects.toThrow( - /lowestIndex is lower than or equal to the last finalized index 10/, - ); + await expect( + taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 10 }], txHash2, 'test'), + ).rejects.toThrow(/Cannot store pending index 10.*lower than or equal to the last finalized index 10/); }); it('allows storing a pending index higher than the last finalized index', async () => { @@ -135,11 +129,13 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); // First store and finalize an index - await taggingStore.storePendingIndexes([range(secret1, 10)], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 10 }], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Store a pending index higher than the finalized index - should succeed - await expect(taggingStore.storePendingIndexes([range(secret1, 15)], txHash2, 'test')).resolves.not.toThrow(); + await expect( + taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 15 }], txHash2, 'test'), + ).resolves.not.toThrow(); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 20, 'test'); expect(txHashes).toHaveLength(1); @@ -154,12 +150,12 @@ describe('SenderTaggingStore', () => { const indexBeyondWindow = finalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; // First store and finalize an index - await taggingStore.storePendingIndexes([range(secret1, finalizedIndex)], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: finalizedIndex }], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Try to store an index beyond the window await expect( - taggingStore.storePendingIndexes([range(secret1, indexBeyondWindow)], txHash2, 'test'), + taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: indexBeyondWindow }], txHash2, 'test'), ).rejects.toThrow( `Highest used index ${indexBeyondWindow} is further than window length from the highest finalized index ${finalizedIndex}`, ); @@ -172,12 +168,12 @@ describe('SenderTaggingStore', () => { const indexAtBoundary = finalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; // First store and finalize an index - await taggingStore.storePendingIndexes([range(secret1, finalizedIndex)], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: finalizedIndex }], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Store an index at the boundary, but check is >, so it should succeed await expect( - taggingStore.storePendingIndexes([range(secret1, indexAtBoundary)], txHash2, 'test'), + taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: indexAtBoundary }], txHash2, 'test'), ).resolves.not.toThrow(); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, indexAtBoundary + 5, 'test'); @@ -198,9 +194,9 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); const txHash3 = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash2, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 8)], txHash3, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 8 }], txHash3, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 4, 9, 'test'); expect(txHashes).toHaveLength(2); @@ -213,8 +209,8 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash1, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 10)], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 10 }], txHash2, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 5, 10, 'test'); expect(txHashes).toHaveLength(1); @@ -227,16 +223,16 @@ describe('SenderTaggingStore', () => { const txHash3 = TxHash.random(); const txHash4 = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash2, 'test'); // We store different secret with txHash1 to check we correctly don't return it in the result - await taggingStore.storePendingIndexes([range(secret2, 7)], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret2, index: 7 }], txHash1, 'test'); // Store "parallel" index for secret1 with a different tx (can happen when sending logs from multiple PXEs) - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash3, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash4, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash3, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash4, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); - // Should have 4 unique tx hashes for secret1 + // Should have 3 unique tx hashes for secret1 expect(txHashes).toEqual(expect.arrayContaining([txHash1, txHash2, txHash3, txHash4])); }); }); @@ -249,7 +245,7 @@ describe('SenderTaggingStore', () => { it('returns the last finalized index after finalizePendingIndexes', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash, 'test'); await taggingStore.finalizePendingIndexes([txHash], 'test'); const lastFinalized = await taggingStore.getLastFinalizedIndex(secret1, 'test'); @@ -265,7 +261,7 @@ describe('SenderTaggingStore', () => { it('returns the last finalized index when no pending indexes exist', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash, 'test'); await taggingStore.finalizePendingIndexes([txHash], 'test'); const lastUsed = await taggingStore.getLastUsedIndex(secret1, 'test'); @@ -277,11 +273,11 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); // First, finalize an index - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Then add a higher pending index - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); const lastUsed = await taggingStore.getLastUsedIndex(secret1, 'test'); expect(lastUsed).toBe(7); @@ -292,9 +288,9 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); const txHash3 = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash3, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash3, 'test'); const lastUsed = await taggingStore.getLastUsedIndex(secret1, 'test'); expect(lastUsed).toBe(7); @@ -306,9 +302,9 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); - await taggingStore.storePendingIndexes([range(secret2, 5)], txHash1, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret2, index: 5 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); await taggingStore.dropPendingIndexes([txHash1], 'test'); @@ -326,7 +322,7 @@ describe('SenderTaggingStore', () => { describe('finalizePendingIndexes', () => { it('moves pending index to finalized for a given tx hash', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash, 'test'); await taggingStore.finalizePendingIndexes([txHash], 'test'); @@ -342,10 +338,10 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); await taggingStore.finalizePendingIndexes([txHash2], 'test'); const lastFinalized = await taggingStore.getLastFinalizedIndex(secret1, 'test'); @@ -357,8 +353,8 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); // Store both pending indexes first - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash1, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash2, 'test'); // Finalize the higher index first await taggingStore.finalizePendingIndexes([txHash1], 'test'); @@ -370,14 +366,14 @@ describe('SenderTaggingStore', () => { expect(lastFinalized).toBe(7); // Should remain at 7 }); - it('prunes pending indexes with lower or equal highestIndex than finalized', async () => { + it('prunes pending indexes with lower or equal index than finalized', async () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); const txHash3 = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash2, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash3, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash3, 'test'); // Finalize txHash2 (index 5) await taggingStore.finalizePendingIndexes([txHash2], 'test'); @@ -391,7 +387,14 @@ describe('SenderTaggingStore', () => { it('handles multiple secrets in the same tx', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3), range(secret2, 7)], txHash, 'test'); + await taggingStore.storePendingIndexes( + [ + { extendedSecret: secret1, index: 3 }, + { extendedSecret: secret2, index: 7 }, + ], + txHash, + 'test', + ); await taggingStore.finalizePendingIndexes([txHash], 'test'); @@ -402,19 +405,9 @@ describe('SenderTaggingStore', () => { expect(lastFinalized2).toBe(7); }); - it('finalizes the highestIndex of a range', async () => { - const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3, 7)], txHash, 'test'); - - await taggingStore.finalizePendingIndexes([txHash], 'test'); - - const lastFinalized = await taggingStore.getLastFinalizedIndex(secret1, 'test'); - expect(lastFinalized).toBe(7); - }); - it('does nothing when tx hash does not exist', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash, 'test'); await taggingStore.finalizePendingIndexes([TxHash.random()], 'test'); @@ -434,7 +427,7 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); // Step 1: Add pending index - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); expect(await taggingStore.getLastUsedIndex(secret1, 'test')).toBe(3); expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBeUndefined(); @@ -444,7 +437,7 @@ describe('SenderTaggingStore', () => { expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(3); // Step 3: Add a new higher pending index - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); expect(await taggingStore.getLastUsedIndex(secret1, 'test')).toBe(7); expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(3); @@ -458,8 +451,8 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 5)], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash2, 'test'); expect(await taggingStore.getLastUsedIndex(secret1, 'test')).toBe(5); @@ -475,14 +468,14 @@ describe('SenderTaggingStore', () => { const txHash3 = TxHash.random(); // Secret1: pending -> finalized - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Secret2: pending (not finalized) - await taggingStore.storePendingIndexes([range(secret2, 5)], txHash2, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret2, index: 5 }], txHash2, 'test'); // Secret1: new pending - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash3, 'test'); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash3, 'test'); expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(3); expect(await taggingStore.getLastUsedIndex(secret1, 'test')).toBe(7); @@ -491,135 +484,18 @@ describe('SenderTaggingStore', () => { }); }); - describe('finalizePendingIndexesOfAPartiallyRevertedTx', () => { - function makeTxEffect(txHash: TxHash, siloedTags: SiloedTag[]): TxEffect { - return new TxEffect( - RevertCode.APP_LOGIC_REVERTED, - txHash, - Fr.ZERO, - [Fr.random()], // noteHashes (at least 1 nullifier required below, not here) - [Fr.random()], // nullifiers (at least 1 required) - [], // l2ToL1Msgs - [], // publicDataWrites - siloedTags.map(tag => PrivateLog.random(tag.value)), // privateLogs with surviving tags - [], // publicLogs - [], // contractClassLogs - ); - } - - it('finalizes only the indexes whose tags appear in TxEffect', async () => { - const txHash = TxHash.random(); - - // Store a range [3, 5] for secret1 in the same tx - await taggingStore.storePendingIndexes([range(secret1, 3, 5)], txHash, 'test'); - - // Compute the siloed tag for index 3 (the one that survives) - const survivingTag = await SiloedTag.compute({ extendedSecret: secret1, index: 3 }); - const txEffect = makeTxEffect(txHash, [survivingTag]); - - await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(txEffect, 'test'); - - // Index 3 should be finalized (it was onchain) - expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(3); - // All pending indexes for this tx should be removed - const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); - expect(txHashes).toHaveLength(0); - }); - - it('drops all indexes when no tags survive onchain', async () => { - const txHash = TxHash.random(); - - await taggingStore.storePendingIndexes([range(secret1, 3, 5)], txHash, 'test'); - - // TxEffect with no matching private logs (empty) - const txEffect = makeTxEffect(txHash, []); - - await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(txEffect, 'test'); - - // No finalized index should be set - expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBeUndefined(); - // All pending indexes for this tx should be removed - const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); - expect(txHashes).toHaveLength(0); - }); - - it('handles multiple secrets affected by the same partially reverted tx', async () => { - const txHash = TxHash.random(); - - // Store pending index ranges for both secrets in the same tx - await taggingStore.storePendingIndexes([range(secret1, 3, 5), range(secret2, 7)], txHash, 'test'); - - // Only index 3 for secret1 survives onchain; other indexes for secret1 and secret2 are dropped - const survivingTag = await SiloedTag.compute({ extendedSecret: secret1, index: 3 }); - const txEffect = makeTxEffect(txHash, [survivingTag]); - - await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(txEffect, 'test'); - - // secret1: index 3 should be finalized - expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(3); - expect(await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test')).toHaveLength(0); - - // secret2: no finalized index, all pending removed - expect(await taggingStore.getLastFinalizedIndex(secret2, 'test')).toBeUndefined(); - expect(await taggingStore.getTxHashesOfPendingIndexes(secret2, 0, 10, 'test')).toHaveLength(0); - }); - - it('preserves pending indexes from other txs', async () => { - const revertedTxHash = TxHash.random(); - const otherTxHash = TxHash.random(); - - // Store pending indexes: one from reverted tx, one from another tx - await taggingStore.storePendingIndexes([range(secret1, 3)], revertedTxHash, 'test'); - await taggingStore.storePendingIndexes([range(secret1, 7)], otherTxHash, 'test'); - - // TxEffect with no surviving tags for the reverted tx - const txEffect = makeTxEffect(revertedTxHash, []); - - await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(txEffect, 'test'); - - // No finalized index (nothing survived from the reverted tx) - expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBeUndefined(); - // The other tx's pending index should still be there - const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); - expect(txHashes).toHaveLength(1); - expect(txHashes[0]).toEqual(otherTxHash); - }); - - it('correctly updates finalized index when there is an existing finalized index', async () => { - const txHash1 = TxHash.random(); - const revertedTxHash = TxHash.random(); - - // Store and finalize index 2 - await taggingStore.storePendingIndexes([range(secret1, 2)], txHash1, 'test'); - await taggingStore.finalizePendingIndexes([txHash1], 'test'); - - // Store a pending range [4, 6] for a partially reverted tx - await taggingStore.storePendingIndexes([range(secret1, 4, 6)], revertedTxHash, 'test'); - - // Only index 4 survives - const survivingTag = await SiloedTag.compute({ extendedSecret: secret1, index: 4 }); - const txEffect = makeTxEffect(revertedTxHash, [survivingTag]); - - await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(txEffect, 'test'); - - // Finalized index should be updated to 4 (higher than previous 2) - expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(4); - expect(await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test')).toHaveLength(0); - }); - }); - describe('staged writes', () => { it('writes of uncommitted jobs are not visible outside the job that makes them', async () => { const committedTxHash = TxHash.random(); { const commitJobId: string = 'commit-job'; - await taggingStore.storePendingIndexes([range(secret1, 3)], committedTxHash, commitJobId); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], committedTxHash, commitJobId); await taggingStore.commit(commitJobId); } const stagedTxHash = TxHash.random(); const stagingJobId: string = 'staging-job'; - await taggingStore.storePendingIndexes([range(secret1, 5)], stagedTxHash, stagingJobId); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], stagedTxHash, stagingJobId); // For a job without any staged data we should only get committed data const txHashesWithoutJobId = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'no-data-job'); @@ -637,7 +513,7 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); { const commitJobId: string = 'commit-job'; - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, commitJobId); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, commitJobId); await taggingStore.finalizePendingIndexes([txHash1], commitJobId); await taggingStore.commit(commitJobId); } @@ -646,7 +522,7 @@ describe('SenderTaggingStore', () => { const stagingJobId: string = 'staging-job'; // Stage a higher finalized index (not committed) - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, stagingJobId); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, stagingJobId); await taggingStore.finalizePendingIndexes([txHash2], stagingJobId); // With a different jobId, should get the committed finalized index @@ -661,8 +537,8 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); const commitJobId: string = 'commit-job'; - await taggingStore.storePendingIndexes([range(secret1, 2)], txHash1, commitJobId); - await taggingStore.storePendingIndexes([range(secret1, 3)], txHash2, commitJobId); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 2 }], txHash1, commitJobId); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash2, commitJobId); await taggingStore.finalizePendingIndexes([txHash1], commitJobId); await taggingStore.commit(commitJobId); } @@ -670,7 +546,7 @@ describe('SenderTaggingStore', () => { const stagingJobId: string = 'staging-job'; { const txHash3 = TxHash.random(); - await taggingStore.storePendingIndexes([range(secret1, 7)], txHash3, stagingJobId); + await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash3, stagingJobId); await taggingStore.finalizePendingIndexes([txHash3], stagingJobId); await taggingStore.discardStaged(stagingJobId); } diff --git a/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.ts b/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.ts index 05f79be89b88..1b15bbbb207a 100644 --- a/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.ts +++ b/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.ts @@ -1,13 +1,10 @@ import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store'; -import { ExtendedDirectionalAppTaggingSecret, SiloedTag, type TaggingIndexRange } from '@aztec/stdlib/logs'; -import { TxEffect, TxHash } from '@aztec/stdlib/tx'; +import type { ExtendedDirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs'; +import { TxHash } from '@aztec/stdlib/tx'; import type { StagedStore } from '../../job_coordinator/job_coordinator.js'; import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/constants.js'; -/** Internal representation of a pending index range entry. */ -type PendingIndexesEntry = { lowestIndex: number; highestIndex: number; txHash: string }; - /** * Data provider of tagging data used when syncing the sender tagging indexes. The recipient counterpart of this class * is called RecipientTaggingStore. We have the data stores separate for sender and recipient because @@ -18,19 +15,20 @@ export class SenderTaggingStore implements StagedStore { #store: AztecAsyncKVStore; - // Stores the pending index ranges for each directional app tagging secret. Pending here means that the tx that - // contained the private logs with tags corresponding to these indexes has not been finalized yet. + // Stores the pending indexes for each directional app tagging secret. Pending here means that the tx that contained + // the private logs with tags corresponding to these indexes has not been finalized yet. + // + // We don't store just the highest index because if their transaction is dropped we'd then need the information about + // the lower pending indexes. For each secret-tx pair however we only store the largest index used in that tx, since + // the smaller ones are irrelevant due to tx atomicity. // - // We store the full range (lowestIndex, highestIndex) for each secret-tx pair because transactions can partially - // revert, in which case only some logs (from the non-revertible phase) survive onchain. By storing the range, - // we can expand it and check each individual siloed tag against the TxEffect to determine which indexes made it - // onchain. + // TODO(#17615): This assumes no logs are used in the non-revertible phase. // - // directional app tagging secret => { lowestIndex, highestIndex, txHash }[] - #pendingIndexes: AztecAsyncMap; + // directional app tagging secret => { pending index, txHash }[] + #pendingIndexes: AztecAsyncMap; - // jobId => directional app tagging secret => { lowestIndex, highestIndex, txHash }[] - #pendingIndexesForJob: Map>; + // jobId => directional app tagging secret => { pending index, txHash }[] + #pendingIndexesForJob: Map>; // Stores the last (highest) finalized index for each directional app tagging secret. We care only about the last // index because unlike the pending indexes, it will never happen that a finalized index would be removed and hence @@ -52,7 +50,7 @@ export class SenderTaggingStore implements StagedStore { this.#lastFinalizedIndexesForJob = new Map(); } - #getPendingIndexesForJob(jobId: string): Map { + #getPendingIndexesForJob(jobId: string): Map { let pendingIndexesForJob = this.#pendingIndexesForJob.get(jobId); if (!pendingIndexesForJob) { pendingIndexesForJob = new Map(); @@ -70,7 +68,7 @@ export class SenderTaggingStore implements StagedStore { return jobStagedLastFinalizedIndexes; } - async #readPendingIndexes(jobId: string, secret: string): Promise { + async #readPendingIndexes(jobId: string, secret: string): Promise<{ index: number; txHash: string }[]> { // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there // are no pending read requests). The staged value still takes precedence if it exists. const dbValue = await this.#pendingIndexes.getAsync(secret); @@ -78,7 +76,7 @@ export class SenderTaggingStore implements StagedStore { return staged !== undefined ? staged : (dbValue ?? []); } - #writePendingIndexes(jobId: string, secret: string, pendingIndexes: PendingIndexesEntry[]) { + #writePendingIndexes(jobId: string, secret: string, pendingIndexes: { index: number; txHash: string }[]) { this.#getPendingIndexesForJob(jobId).set(secret, pendingIndexes); } @@ -128,37 +126,57 @@ export class SenderTaggingStore implements StagedStore { } /** - * Stores pending index ranges. - * @remarks If the same (secret, txHash) pair already exists in the db with an equal range, it's a no-op. This is - * expected to happen because whenever we start sync we start from the last finalized index and we can have pending - * ranges already stored from previous syncs. If the ranges differ, it throws an error as that indicates a bug. - * @param ranges - The tagging index ranges containing the directional app tagging secrets and the index ranges that are - * to be stored in the db. - * @param txHash - The tx in which the tagging indexes were used in private logs. + * Stores pending indexes. + * @remarks Ignores the index if the same preTag + txHash combination already exists in the db with the same index. + * This is expected to happen because whenever we start sync we start from the last finalized index and we can have + * pending indexes already stored from previous syncs. + * @param preTags - The pre-tags containing the directional app tagging secrets and the indexes that are to be + * stored in the db. + * @param txHash - The tx in which the pretags were used in private logs. * @param jobId - job context for staged writes to this store. See `JobCoordinator` for more details. - * @throws If the highestIndex is further than window length from the highest finalized index for the same secret. - * @throws If the lowestIndex is lower than or equal to the last finalized index for the same secret. - * @throws If a different range already exists for the same (secret, txHash) pair. + * @throws If any two pre-tags contain the same directional app tagging secret. This is enforced because we care + * only about the highest index for a given secret that was used in the tx. Hence this check is a good way to catch + * bugs. + * @throws If the newly stored pending index is further than window length from the highest finalized index for the + * same secret. This is enforced in order to give a guarantee to a recipient that he doesn't need to look further than + * window length ahead of the highest finalized index. + * @throws If a secret + txHash pair already exists in the db with a different index value. It should never happen + * that we would attempt to store a different index for a given secret-txHash pair because we always store just the + * highest index for a given secret-txHash pair. Hence this is a good way to catch bugs. + * @throws If the newly stored pending index is lower than or equal to the last finalized index for the same secret. + * This is enforced because this should never happen if the syncing is done correctly as we look for logs from higher + * indexes than finalized ones. */ - storePendingIndexes(ranges: TaggingIndexRange[], txHash: TxHash, jobId: string): Promise { - if (ranges.length === 0) { + storePendingIndexes(preTags: PreTag[], txHash: TxHash, jobId: string): Promise { + if (preTags.length === 0) { return Promise.resolve(); } + // The secrets in pre-tags should be unique because we always store just the highest index per given secret-txHash + // pair. Below we check that this is the case. + const secretsSet = new Set(preTags.map(preTag => preTag.extendedSecret.toString())); + if (secretsSet.size !== preTags.length) { + return Promise.reject(new Error(`Duplicate secrets found when storing pending indexes`)); + } + const txHashStr = txHash.toString(); return this.#store.transactionAsync(async () => { // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive - const rangeReadPromises = ranges.map(range => ({ - range, - secretStr: range.extendedSecret.toString(), - pending: this.#readPendingIndexes(jobId, range.extendedSecret.toString()), - finalized: this.#readLastFinalizedIndex(jobId, range.extendedSecret.toString()), - })); + const preTagReadPromises = preTags.map(({ extendedSecret, index }) => { + const secretStr = extendedSecret.toString(); + return { + extendedSecret, + secretStr, + index, + pending: this.#readPendingIndexes(jobId, secretStr), + finalized: this.#readLastFinalizedIndex(jobId, secretStr), + }; + }); // Await all reads together - const rangeData = await Promise.all( - rangeReadPromises.map(async item => ({ + const preTagData = await Promise.all( + preTagReadPromises.map(async item => ({ ...item, pendingData: await item.pending, finalizedIndex: await item.finalized, @@ -166,51 +184,48 @@ export class SenderTaggingStore implements StagedStore { ); // Process in memory and validate - for (const { range, secretStr, pendingData, finalizedIndex } of rangeData) { - // Check that the highest index is not further than window length from the highest finalized index. - if (range.highestIndex > (finalizedIndex ?? 0) + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) { + for (const { secretStr, index, pendingData, finalizedIndex } of preTagData) { + // First we check that for any secret the highest used index in tx is not further than window length from + // the highest finalized index. + if (index > (finalizedIndex ?? 0) + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) { throw new Error( - `Highest used index ${range.highestIndex} is further than window length from the highest finalized index ${finalizedIndex ?? 0}. + `Highest used index ${index} is further than window length from the highest finalized index ${finalizedIndex ?? 0}. Tagging window length ${UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN} is configured too low. Contact the Aztec team to increase it!`, ); } - // Throw if the lowest index is lower than or equal to the last finalized index - if (finalizedIndex !== undefined && range.lowestIndex <= finalizedIndex) { + // Throw if the new pending index is lower than or equal to the last finalized index + if (finalizedIndex !== undefined && index <= finalizedIndex) { throw new Error( - `Cannot store pending index range [${range.lowestIndex}, ${range.highestIndex}] for secret ${secretStr}: ` + - `lowestIndex is lower than or equal to the last finalized index ${finalizedIndex}`, + `Cannot store pending index ${index} for secret ${secretStr}: ` + + `it is lower than or equal to the last finalized index ${finalizedIndex}`, ); } - // Check if an entry with the same txHash already exists - const existingEntry = pendingData.find(entry => entry.txHash === txHashStr); + // Check if this secret + txHash combination already exists + const existingForSecretAndTx = pendingData.find(entry => entry.txHash === txHashStr); - if (existingEntry) { - // Assert that the ranges are equal — different ranges for the same (secret, txHash) indicates a bug - if (existingEntry.lowestIndex !== range.lowestIndex || existingEntry.highestIndex !== range.highestIndex) { + if (existingForSecretAndTx) { + // If it exists with a different index, throw an error + if (existingForSecretAndTx.index !== index) { throw new Error( - `Conflicting range for secret ${secretStr} and txHash ${txHashStr}: ` + - `existing [${existingEntry.lowestIndex}, ${existingEntry.highestIndex}] vs ` + - `new [${range.lowestIndex}, ${range.highestIndex}]`, + `Cannot store index ${index} for secret ${secretStr} and txHash ${txHashStr}: ` + + `a different index ${existingForSecretAndTx.index} already exists for this secret-txHash pair`, ); } - // Exact duplicate — skip + // If it exists with the same index, ignore the update (no-op) } else { - this.#writePendingIndexes(jobId, secretStr, [ - ...pendingData, - { lowestIndex: range.lowestIndex, highestIndex: range.highestIndex, txHash: txHashStr }, - ]); + // If it doesn't exist, add it + this.#writePendingIndexes(jobId, secretStr, [...pendingData, { index, txHash: txHashStr }]); } } }); } /** - * Returns the transaction hashes of all pending transactions that contain highest indexes within a specified range - * for a given directional app tagging secret. We check based on the highest indexes only as that is the relevant - * information for the caller of this function. + * Returns the transaction hashes of all pending transactions that contain indexes within a specified range + * for a given directional app tagging secret. * @param secret - The directional app tagging secret to query pending indexes for. * @param startIndex - The lower bound of the index range (inclusive). * @param endIndex - The upper bound of the index range (exclusive). @@ -226,7 +241,7 @@ export class SenderTaggingStore implements StagedStore { return this.#store.transactionAsync(async () => { const existing = await this.#readPendingIndexes(jobId, secret.toString()); const txHashes = existing - .filter(entry => entry.highestIndex >= startIndex && entry.highestIndex < endIndex) + .filter(entry => entry.index >= startIndex && entry.index < endIndex) .map(entry => entry.txHash); return Array.from(new Set(txHashes)).map(TxHash.fromString); }); @@ -254,15 +269,16 @@ export class SenderTaggingStore implements StagedStore { const pendingPromise = this.#readPendingIndexes(jobId, secretStr); const finalizedPromise = this.#readLastFinalizedIndex(jobId, secretStr); - const [pendingEntries, lastFinalized] = await Promise.all([pendingPromise, finalizedPromise]); + const [pendingTxScopedIndexes, lastFinalized] = await Promise.all([pendingPromise, finalizedPromise]); + const pendingIndexes = pendingTxScopedIndexes.map(entry => entry.index); - if (pendingEntries.length === 0) { + if (pendingTxScopedIndexes.length === 0) { return lastFinalized; } - // As the last used index we return the highest one from the pending index ranges. Note that this value will be - // always higher than the last finalized index because we prune lower pending indexes when a tx is finalized. - return Math.max(...pendingEntries.map(entry => entry.highestIndex)); + // As the last used index we return the highest one from the pending indexes. Note that this value will be always + // higher than the last finalized index because we prune lower pending indexes when a tx is finalized. + return Math.max(...pendingIndexes); }); } @@ -278,7 +294,7 @@ export class SenderTaggingStore implements StagedStore { return this.#store.transactionAsync(async () => { // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive - const secretReadPromises: Map> = new Map(); + const secretReadPromises: Map> = new Map(); for await (const secret of this.#pendingIndexes.keysAsync()) { secretReadPromises.set(secret, this.#readPendingIndexes(jobId, secret)); @@ -314,15 +330,22 @@ export class SenderTaggingStore implements StagedStore { }); } - /** Prefetches all pending and finalized index data for every secret (from both DB and staged writes). */ - #getSecretsWithPendingData( - jobId: string, - ): Promise<{ secret: string; pendingData: PendingIndexesEntry[]; lastFinalized: number | undefined }[]> { + /** + * Updates pending indexes corresponding to the given transaction hashes to be finalized and prunes any lower pending + * indexes. + */ + finalizePendingIndexes(txHashes: TxHash[], jobId: string): Promise { + if (txHashes.length === 0) { + return Promise.resolve(); + } + + const txHashStrings = new Set(txHashes.map(tx => tx.toString())); + return this.#store.transactionAsync(async () => { // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive const secretDataPromises: Map< string, - { pending: Promise; finalized: Promise } + { pending: Promise<{ index: number; txHash: string }[]>; finalized: Promise } > = new Map(); for await (const secret of this.#pendingIndexes.keysAsync()) { @@ -352,125 +375,55 @@ export class SenderTaggingStore implements StagedStore { })), ); - return dataResults.filter(r => r.pendingData.length > 0); - }); - } - - /** - * Updates pending indexes corresponding to the given transaction hashes to be finalized and prunes any lower pending - * indexes. - */ - async finalizePendingIndexes(txHashes: TxHash[], jobId: string): Promise { - if (txHashes.length === 0) { - return; - } - - const txHashStrings = new Set(txHashes.map(tx => tx.toString())); - const secretsWithData = await this.#getSecretsWithPendingData(jobId); - - for (const { secret, pendingData, lastFinalized } of secretsWithData) { - let currentPending = pendingData; - let currentFinalized = lastFinalized; - - // Process all txHashes for this secret - for (const txHashStr of txHashStrings) { - const matchingEntries = currentPending.filter(item => item.txHash === txHashStr); - if (matchingEntries.length === 0) { - // This is expected as a higher index might have already been finalized which would lead to pruning of - // pending entries. + // Process all txHashes for each secret in memory + for (const { secret, pendingData, lastFinalized } of dataResults) { + if (!pendingData || pendingData.length === 0) { continue; } - if (matchingEntries.length > 1) { - // We should always just store the highest pending index for a given tx hash and secret because the lower - // values are irrelevant. - throw new Error(`Multiple pending entries found for tx hash ${txHashStr} and secret ${secret}`); - } - - const newFinalized = matchingEntries[0].highestIndex; - - if (newFinalized < (currentFinalized ?? 0)) { - // This should never happen because when last finalized index was finalized we should have pruned the lower - // pending indexes. - throw new Error( - `New finalized index ${newFinalized} is smaller than the current last finalized index ${currentFinalized}`, - ); - } - - currentFinalized = newFinalized; - - // When we add pending indexes, we ensure they are higher than the last finalized index. However, because we - // cannot control the order in which transactions are finalized, there may be pending indexes that are now - // obsolete because they are lower than the most recently finalized index. For this reason, we prune these - // outdated pending indexes. - currentPending = currentPending.filter(item => item.highestIndex > currentFinalized!); - } - - // Write final state if changed - if (currentFinalized !== lastFinalized) { - this.#writeLastFinalizedIndex(jobId, secret, currentFinalized!); - } - if (currentPending !== pendingData) { - this.#writePendingIndexes(jobId, secret, currentPending); - } - } - } - - /** - * Handles finalization of pending indexes for a transaction whose execution was partially reverted. - * Recomputes the siloed tags for each pending index of the given tx and checks which ones appear in the - * TxEffect's private logs (i.e., which ones made it onchain). Those that survived are finalized; those that - * didn't are dropped. - * @param txEffect - The tx effect of the partially reverted transaction. - * @param jobId - job context for staged writes to this store. See `JobCoordinator` for more details. - */ - async finalizePendingIndexesOfAPartiallyRevertedTx(txEffect: TxEffect, jobId: string): Promise { - const txHashStr = txEffect.txHash.toString(); - - // Build a set of all siloed tag values that made it onchain (first field of each private log). - const onChainTags = new Set(txEffect.privateLogs.map(log => log.fields[0].toString())); + let currentPending = pendingData; + let currentFinalized = lastFinalized; - const secretsWithData = await this.#getSecretsWithPendingData(jobId); + // Process all txHashes for this secret + for (const txHashStr of txHashStrings) { + const matchingIndexes = currentPending.filter(item => item.txHash === txHashStr).map(item => item.index); + if (matchingIndexes.length === 0) { + continue; + } - for (const { secret, pendingData, lastFinalized } of secretsWithData) { - const matchingEntries = pendingData.filter(item => item.txHash === txHashStr); - if (matchingEntries.length === 0) { - // This is expected as a higher index might have already been finalized which would lead to pruning of - // pending entries. - continue; - } + if (matchingIndexes.length > 1) { + // We should always just store the highest pending index for a given tx hash and secret because the lower + // values are irrelevant. + throw new Error(`Multiple pending indexes found for tx hash ${txHashStr} and secret ${secret}`); + } - if (matchingEntries.length > 1) { - // We should always just store the highest pending index for a given tx hash and secret because the lower - // values are irrelevant. - throw new Error(`Multiple pending entries found for tx hash ${txHashStr} and secret ${secret}`); - } + const newFinalized = matchingIndexes[0]; - const pendingEntry = matchingEntries[0]; + if (newFinalized < (currentFinalized ?? 0)) { + // This should never happen because when last finalized index was finalized we should have pruned the lower + // pending indexes. + throw new Error( + `New finalized index ${newFinalized} is smaller than the current last finalized index ${currentFinalized}`, + ); + } - // Expand each matching entry's range and recompute siloed tags for each index. - const extendedSecret = ExtendedDirectionalAppTaggingSecret.fromString(secret); - let highestSurvivingIndex: number | undefined; + currentFinalized = newFinalized; - for (let index = pendingEntry.lowestIndex; index <= pendingEntry.highestIndex; index++) { - const siloedTag = await SiloedTag.compute({ extendedSecret, index }); - if (onChainTags.has(siloedTag.value.toString())) { - highestSurvivingIndex = highestSurvivingIndex !== undefined ? Math.max(highestSurvivingIndex, index) : index; + // When we add pending indexes, we ensure they are higher than the last finalized index. However, because we + // cannot control the order in which transactions are finalized, there may be pending indexes that are now + // obsolete because they are lower than the most recently finalized index. For this reason, we prune these + // outdated pending indexes. + currentPending = currentPending.filter(item => item.index > currentFinalized!); } - } - // Remove all entries for this txHash from pending (both surviving and non-surviving). - let currentPending = pendingData.filter(item => item.txHash !== txHashStr); - - if (highestSurvivingIndex !== undefined) { - const newFinalized = Math.max(lastFinalized ?? 0, highestSurvivingIndex); - this.#writeLastFinalizedIndex(jobId, secret, newFinalized); - - // Prune pending indexes that are now <= the finalized index. - currentPending = currentPending.filter(item => item.highestIndex > newFinalized); + // Write final state if changed + if (currentFinalized !== lastFinalized) { + this.#writeLastFinalizedIndex(jobId, secret, currentFinalized!); + } + if (currentPending !== pendingData) { + this.#writePendingIndexes(jobId, secret, currentPending); + } } - - this.#writePendingIndexes(jobId, secret, currentPending); - } + }); } } diff --git a/yarn-project/pxe/src/tagging/index.ts b/yarn-project/pxe/src/tagging/index.ts index 6b812a8f0a47..ea8c6f80f613 100644 --- a/yarn-project/pxe/src/tagging/index.ts +++ b/yarn-project/pxe/src/tagging/index.ts @@ -16,4 +16,4 @@ export { getAllPrivateLogsByTags, getAllPublicLogsByTagsFromContract } from './g // Re-export tagging-related types from stdlib export { ExtendedDirectionalAppTaggingSecret, Tag, SiloedTag } from '@aztec/stdlib/logs'; -export { type PreTag, type TaggingIndexRange } from '@aztec/stdlib/logs'; +export { type PreTag } from '@aztec/stdlib/logs'; diff --git a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts index dedfacbf5dda..d214b6e50120 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts @@ -1,12 +1,10 @@ import { BlockNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; -import { RevertCode } from '@aztec/stdlib/avm'; import { BlockHash } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; -import { PrivateLog } from '@aztec/stdlib/logs'; import { randomExtendedDirectionalAppTaggingSecret, randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; -import { type IndexedTxEffect, TxEffect, TxExecutionResult, TxHash, TxReceipt, TxStatus } from '@aztec/stdlib/tx'; +import { TxExecutionResult, TxHash, TxReceipt, TxStatus } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -277,68 +275,4 @@ describe('syncSenderTaggingIndexes', () => { expect(await taggingStore.getLastFinalizedIndex(secret, 'test')).toBe(pendingAndFinalizedIndex); expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(pendingAndFinalizedIndex); }); - - it('handles a partially reverted transaction', async () => { - await setUp(); - - const revertedTxHash = TxHash.random(); - - // Create logs at indexes 4 and 6 for the same (reverted) tx - const tag4 = await computeSiloedTagForIndex(4); - const tag6 = await computeSiloedTagForIndex(6); - - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { - return Promise.resolve( - tags.map((tag: SiloedTag) => { - if (tag.equals(tag4)) { - return [makeLog(revertedTxHash, tag4.value)]; - } else if (tag.equals(tag6)) { - return [makeLog(revertedTxHash, tag6.value)]; - } - return []; - }), - ); - }); - - // Mock getTxReceipt to return FINALIZED with APP_LOGIC_REVERTED - aztecNode.getTxReceipt.mockResolvedValue( - new TxReceipt( - revertedTxHash, - TxStatus.FINALIZED, - TxExecutionResult.APP_LOGIC_REVERTED, - undefined, - undefined, - undefined, - BlockNumber(14), - ), - ); - - // Mock getTxEffect to return a TxEffect where only the tag at index 4 survived (non-revertible phase) - const txEffect = new TxEffect( - RevertCode.APP_LOGIC_REVERTED, - revertedTxHash, - Fr.ZERO, - [Fr.random()], // noteHashes - [Fr.random()], // nullifiers - [], // l2ToL1Msgs - [], // publicDataWrites - [PrivateLog.random(tag4.value)], // only the tag at index 4 survived - [], // publicLogs - [], // contractClassLogs - ); - - aztecNode.getTxEffect.mockResolvedValue({ - data: txEffect, - l2BlockNumber: BlockNumber(14), - l2BlockHash: MOCK_ANCHOR_BLOCK_HASH, - txIndexInBlock: 0, - } as IndexedTxEffect); - - await syncSenderTaggingIndexes(secret, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - - // Index 4 should be finalized (it survived the partial revert) - expect(await taggingStore.getLastFinalizedIndex(secret, 'test')).toBe(4); - // No pending indexes should remain for this secret - expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(4); - }); }); diff --git a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.ts b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.ts index 516dc00483ef..87d56d6a46e7 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.ts @@ -62,29 +62,11 @@ export async function syncSenderTaggingIndexes( break; } - const { txHashesToFinalize, txHashesToDrop, txHashesWithExecutionReverted } = await getStatusChangeOfPending( - pendingTxHashes, - aztecNode, - ); + const { txHashesToFinalize, txHashesToDrop } = await getStatusChangeOfPending(pendingTxHashes, aztecNode); await taggingStore.dropPendingIndexes(txHashesToDrop, jobId); await taggingStore.finalizePendingIndexes(txHashesToFinalize, jobId); - if (txHashesWithExecutionReverted.length > 0) { - const indexedTxEffects = await Promise.all( - txHashesWithExecutionReverted.map(txHash => aztecNode.getTxEffect(txHash)), - ); - for (const indexedTxEffect of indexedTxEffects) { - if (indexedTxEffect === undefined) { - throw new Error( - 'TxEffect not found for execution-reverted tx. This is either a bug or a reorg has occurred.', - ); - } - - await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(indexedTxEffect.data, jobId); - } - } - // We check if the finalized index has been updated. newFinalizedIndex = await taggingStore.getLastFinalizedIndex(secret, jobId); if (previousFinalizedIndex !== newFinalizedIndex) { diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts index 676b491d8910..7fd0fc92e3f3 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts @@ -51,41 +51,11 @@ describe('getStatusChangeOfPending', () => { ), ); } else if (hash.equals(appLogicRevertedTxHash)) { - return Promise.resolve( - new TxReceipt( - hash, - TxStatus.FINALIZED, - TxExecutionResult.APP_LOGIC_REVERTED, - undefined, - undefined, - undefined, - BlockNumber(10), - ), - ); + return Promise.resolve(new TxReceipt(hash, TxStatus.PROPOSED, TxExecutionResult.APP_LOGIC_REVERTED, undefined)); } else if (hash.equals(teardownRevertedTxHash)) { - return Promise.resolve( - new TxReceipt( - hash, - TxStatus.FINALIZED, - TxExecutionResult.TEARDOWN_REVERTED, - undefined, - undefined, - undefined, - BlockNumber(10), - ), - ); + return Promise.resolve(new TxReceipt(hash, TxStatus.PROPOSED, TxExecutionResult.TEARDOWN_REVERTED, undefined)); } else if (hash.equals(bothRevertedTxHash)) { - return Promise.resolve( - new TxReceipt( - hash, - TxStatus.FINALIZED, - TxExecutionResult.BOTH_REVERTED, - undefined, - undefined, - undefined, - BlockNumber(10), - ), - ); + return Promise.resolve(new TxReceipt(hash, TxStatus.PROPOSED, TxExecutionResult.BOTH_REVERTED, undefined)); } else { throw new Error(`Unexpected tx hash: ${hash.toString()}`); } @@ -104,8 +74,8 @@ describe('getStatusChangeOfPending', () => { ); expect(result.txHashesToFinalize).toEqual([finalizedTxHash]); - expect(result.txHashesToDrop).toEqual([droppedTxHash]); - expect(result.txHashesWithExecutionReverted).toEqual([ + expect(result.txHashesToDrop).toEqual([ + droppedTxHash, appLogicRevertedTxHash, teardownRevertedTxHash, bothRevertedTxHash, @@ -131,7 +101,6 @@ describe('getStatusChangeOfPending', () => { expect(result.txHashesToFinalize).toEqual([txHash]); expect(result.txHashesToDrop).toEqual([]); - expect(result.txHashesWithExecutionReverted).toEqual([]); }); it('does not finalize tx that is only proven', async () => { @@ -154,6 +123,5 @@ describe('getStatusChangeOfPending', () => { // Not finalized yet, so stays pending expect(result.txHashesToFinalize).toEqual([]); expect(result.txHashesToDrop).toEqual([]); - expect(result.txHashesWithExecutionReverted).toEqual([]); }); }); diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts index 1fc434d10c35..8400b16237f3 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts @@ -2,50 +2,35 @@ import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { TxHash, TxStatus } from '@aztec/stdlib/tx'; /** - * Based on receipts obtained from `aztecNode` returns which pending transactions changed their status to finalized, - * dropped, or execution-reverted (but mined). + * Based on receipts obtained from `aztecNode` returns which pending transactions changed their status to finalized or + * dropped. */ export async function getStatusChangeOfPending( pending: TxHash[], aztecNode: AztecNode, -): Promise<{ - txHashesToFinalize: TxHash[]; - txHashesToDrop: TxHash[]; - txHashesWithExecutionReverted: TxHash[]; -}> { +): Promise<{ txHashesToFinalize: TxHash[]; txHashesToDrop: TxHash[] }> { // Get receipts for all pending tx hashes. const receipts = await Promise.all(pending.map(pendingTxHash => aztecNode.getTxReceipt(pendingTxHash))); const txHashesToFinalize: TxHash[] = []; const txHashesToDrop: TxHash[] = []; - const txHashesWithExecutionReverted: TxHash[] = []; for (let i = 0; i < receipts.length; i++) { const receipt = receipts[i]; const txHash = pending[i]; - if (receipt.status === TxStatus.FINALIZED) { - // Tx has been included in a block and the corresponding block is finalized - if (receipt.hasExecutionSucceeded()) { - // No part of execution reverted - we just finalize all the indexes. - txHashesToFinalize.push(txHash); - } else if (receipt.hasExecutionReverted()) { - // Tx was mined but execution reverted (app logic, teardown, or both). Some logs from the non-revertible - // phase may still be onchain. We check which tags made it onchain and finalize those; drop the rest. - txHashesWithExecutionReverted.push(txHash); - } else { - // Defensive check - this branch should never be triggered - throw new Error( - 'Both hasExecutionSucceeded and hasExecutionReverted on the receipt returned false. This should never happen and it implies a bug. Please open an issue.', - ); - } - } else if (receipt.isDropped()) { - // Tx was dropped from the mempool --> we drop the corresponding pending indexes. + if (receipt.status === TxStatus.FINALIZED && receipt.hasExecutionSucceeded()) { + // Tx has been included in a block and the corresponding block is finalized --> we mark the indexes as + // finalized. + txHashesToFinalize.push(txHash); + } else if (receipt.isDropped() || receipt.hasExecutionReverted()) { + // Tx was dropped or reverted --> we drop the corresponding pending indexes. + // TODO(#17615): Don't drop pending indexes corresponding to non-revertible phases. txHashesToDrop.push(txHash); } else { // Tx is still pending, not yet finalized, or was mined successfully but not yet finalized --> we don't do anything. } } - return { txHashesToFinalize, txHashesToDrop, txHashesWithExecutionReverted }; + return { txHashesToFinalize, txHashesToDrop }; } diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts index 572ef56fb88e..789c67c79f8f 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts @@ -1,4 +1,5 @@ import type { Fr } from '@aztec/foundation/curves/bn254'; +import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { BlockHash } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { type ExtendedDirectionalAppTaggingSecret, SiloedTag } from '@aztec/stdlib/logs'; @@ -7,15 +8,17 @@ import { TxHash } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; -import type { SenderTaggingStore } from '../../../storage/tagging_store/sender_tagging_store.js'; +import { SenderTaggingStore } from '../../../storage/tagging_store/sender_tagging_store.js'; import { loadAndStoreNewTaggingIndexes } from './load_and_store_new_tagging_indexes.js'; const MOCK_ANCHOR_BLOCK_HASH = BlockHash.random(); describe('loadAndStoreNewTaggingIndexes', () => { + // Secret to be used on the input of the loadAndStoreNewTaggingIndexes function. let secret: ExtendedDirectionalAppTaggingSecret; + let aztecNode: MockProxy; - let taggingStore: MockProxy; + let taggingStore: SenderTaggingStore; function computeSiloedTagForIndex(index: number) { return SiloedTag.compute({ extendedSecret: secret, index }); @@ -27,21 +30,30 @@ describe('loadAndStoreNewTaggingIndexes', () => { beforeAll(async () => { secret = await randomExtendedDirectionalAppTaggingSecret(); + aztecNode = mock(); }); - beforeEach(() => { - aztecNode = mock(); - taggingStore = mock(); + // Unlike for secret, app address and aztecNode we need a fresh instance of the tagging data provider for each test. + beforeEach(async () => { + aztecNode.getPrivateLogsByTags.mockReset(); + taggingStore = new SenderTaggingStore(await openTmpStore('test')); }); it('no logs found for the given window', async () => { aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { - return Promise.resolve(tags.map(() => [])); + // No log found for any tag + return Promise.resolve(tags.map((_tag: SiloedTag) => [])); }); await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - expect(taggingStore.storePendingIndexes).not.toHaveBeenCalled(); + // Verify that no pending indexes were stored + expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBeUndefined(); + expect(await taggingStore.getLastFinalizedIndex(secret, 'test')).toBeUndefined(); + + // Verify the entire window has no pending tx hashes + const txHashesInWindow = await taggingStore.getTxHashesOfPendingIndexes(secret, 0, 10, 'test'); + expect(txHashesInWindow).toHaveLength(0); }); it('single log found at a specific index', async () => { @@ -55,15 +67,16 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(1); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( - [{ extendedSecret: secret, lowestIndex: index, highestIndex: index }], - txHash, - 'test', - ); + // Verify that the pending index was stored for this txHash + const txHashesInRange = await taggingStore.getTxHashesOfPendingIndexes(secret, index, index + 1, 'test'); + expect(txHashesInRange).toHaveLength(1); + expect(txHashesInRange[0].equals(txHash)).toBe(true); + + // Verify the last used index is correct + expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(index); }); - it('for multiple logs with same txHash stores full index range', async () => { + it('for multiple logs with same txHash stores the highest index', async () => { const txHash = TxHash.random(); const index1 = 3; const index2 = 7; @@ -85,12 +98,17 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(1); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( - [{ extendedSecret: secret, lowestIndex: index1, highestIndex: index2 }], - txHash, - 'test', - ); + // Verify that only the highest index (7) was stored for this txHash and secret + const txHashesAtIndex2 = await taggingStore.getTxHashesOfPendingIndexes(secret, index2, index2 + 1, 'test'); + expect(txHashesAtIndex2).toHaveLength(1); + expect(txHashesAtIndex2[0].equals(txHash)).toBe(true); + + // Verify the lower index is not stored separately + const txHashesAtIndex1 = await taggingStore.getTxHashesOfPendingIndexes(secret, index1, index1 + 1, 'test'); + expect(txHashesAtIndex1).toHaveLength(0); + + // Verify the last used index is the highest + expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(index2); }); it('multiple logs with different txHashes', async () => { @@ -116,17 +134,17 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(2); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( - [{ extendedSecret: secret, lowestIndex: index1, highestIndex: index1 }], - txHash1, - 'test', - ); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( - [{ extendedSecret: secret, lowestIndex: index2, highestIndex: index2 }], - txHash2, - 'test', - ); + // Verify that both txHashes have their respective indexes stored + const txHashesAtIndex1 = await taggingStore.getTxHashesOfPendingIndexes(secret, index1, index1 + 1, 'test'); + expect(txHashesAtIndex1).toHaveLength(1); + expect(txHashesAtIndex1[0].equals(txHash1)).toBe(true); + + const txHashesAtIndex2 = await taggingStore.getTxHashesOfPendingIndexes(secret, index2, index2 + 1, 'test'); + expect(txHashesAtIndex2).toHaveLength(1); + expect(txHashesAtIndex2[0].equals(txHash2)).toBe(true); + + // Verify the last used index is the highest + expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(index2); }); // Expected to happen if sending logs from multiple PXEs at a similar time. @@ -144,17 +162,15 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(2); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( - [{ extendedSecret: secret, lowestIndex: index, highestIndex: index }], - txHash1, - 'test', - ); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( - [{ extendedSecret: secret, lowestIndex: index, highestIndex: index }], - txHash2, - 'test', - ); + // Verify that both txHashes have the same index stored + const txHashesAtIndex = await taggingStore.getTxHashesOfPendingIndexes(secret, index, index + 1, 'test'); + expect(txHashesAtIndex).toHaveLength(2); + const txHashStrings = txHashesAtIndex.map(h => h.toString()); + expect(txHashStrings).toContain(txHash1.toString()); + expect(txHashStrings).toContain(txHash2.toString()); + + // Verify the last used index is correct + expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(index); }); it('complex scenario: multiple txHashes with multiple indexes', async () => { @@ -162,11 +178,10 @@ describe('loadAndStoreNewTaggingIndexes', () => { const txHash2 = TxHash.random(); const txHash3 = TxHash.random(); - // txHash1 has logs at index 1, 2 and 8 → range [1, 8] - // txHash2 has logs at index 3 and 5 → range [3, 5] - // txHash3 has a log at index 9 → range [9, 9] + // txHash1 has logs at index 1 and 8 (should store 8) + // txHash2 has logs at index 3 and 5 (should store 5) + // txHash3 has a log at index 9 (should store 9) const tag1 = await computeSiloedTagForIndex(1); - const tag2 = await computeSiloedTagForIndex(2); const tag3 = await computeSiloedTagForIndex(3); const tag5 = await computeSiloedTagForIndex(5); const tag8 = await computeSiloedTagForIndex(8); @@ -177,8 +192,6 @@ describe('loadAndStoreNewTaggingIndexes', () => { tags.map((t: SiloedTag) => { if (t.equals(tag1)) { return [makeLog(txHash1, tag1.value)]; - } else if (t.equals(tag2)) { - return [makeLog(txHash1, tag1.value)]; } else if (t.equals(tag3)) { return [makeLog(txHash2, tag3.value)]; } else if (t.equals(tag5)) { @@ -195,22 +208,27 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(3); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( - [{ extendedSecret: secret, lowestIndex: 1, highestIndex: 8 }], - txHash1, - 'test', - ); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( - [{ extendedSecret: secret, lowestIndex: 3, highestIndex: 5 }], - txHash2, - 'test', - ); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( - [{ extendedSecret: secret, lowestIndex: 9, highestIndex: 9 }], - txHash3, - 'test', - ); + // Verify txHash1 has highest index 8 (should not be at index 1) + const txHashesAtIndex1 = await taggingStore.getTxHashesOfPendingIndexes(secret, 1, 2, 'test'); + expect(txHashesAtIndex1).toHaveLength(0); + const txHashesAtIndex8 = await taggingStore.getTxHashesOfPendingIndexes(secret, 8, 9, 'test'); + expect(txHashesAtIndex8).toHaveLength(1); + expect(txHashesAtIndex8[0].equals(txHash1)).toBe(true); + + // Verify txHash2 has highest index 5 (should not be at index 3) + const txHashesAtIndex3 = await taggingStore.getTxHashesOfPendingIndexes(secret, 3, 4, 'test'); + expect(txHashesAtIndex3).toHaveLength(0); + const txHashesAtIndex5 = await taggingStore.getTxHashesOfPendingIndexes(secret, 5, 6, 'test'); + expect(txHashesAtIndex5).toHaveLength(1); + expect(txHashesAtIndex5[0].equals(txHash2)).toBe(true); + + // Verify txHash3 has index 9 + const txHashesAtIndex9 = await taggingStore.getTxHashesOfPendingIndexes(secret, 9, 10, 'test'); + expect(txHashesAtIndex9).toHaveLength(1); + expect(txHashesAtIndex9[0].equals(txHash3)).toBe(true); + + // Verify the last used index is the highest + expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(9); }); it('start is inclusive and end is exclusive', async () => { @@ -238,12 +256,16 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, start, end, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - // Only the log at start should be stored; end is exclusive - expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(1); - expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( - [{ extendedSecret: secret, lowestIndex: start, highestIndex: start }], - txHashAtStart, - 'test', - ); + // Verify that the log at start (inclusive) was processed + const txHashesAtStart = await taggingStore.getTxHashesOfPendingIndexes(secret, start, start + 1, 'test'); + expect(txHashesAtStart).toHaveLength(1); + expect(txHashesAtStart[0].equals(txHashAtStart)).toBe(true); + + // Verify that the log at end (exclusive) was NOT processed + const txHashesAtEnd = await taggingStore.getTxHashesOfPendingIndexes(secret, end, end + 1, 'test'); + expect(txHashesAtEnd).toHaveLength(0); + + // Verify the last used index is the start index (since end was not processed) + expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(start); }); }); diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts index 3979f5007189..5558c1097cba 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts @@ -16,7 +16,6 @@ import { getAllPrivateLogsByTags } from '../../get_all_logs_by_tags.js'; * @param end - The ending index (exclusive) of the window to process. * @param aztecNode - The Aztec node instance to query for logs. * @param taggingStore - The data provider to store pending indexes. - * @param anchorBlockHash - Hash of a block to use as reference block when querying node. * @param jobId - Job identifier, used to keep writes in-memory until they can be persisted in a data integrity * preserving way. */ @@ -35,13 +34,12 @@ export async function loadAndStoreNewTaggingIndexes( ); const txsForTags = await getTxsContainingTags(siloedTagsForWindow, aztecNode, anchorBlockHash); - const txIndexesMap = getTxIndexesMap(txsForTags, start, siloedTagsForWindow.length); + const highestIndexMap = getTxHighestIndexMap(txsForTags, start, siloedTagsForWindow.length); - // Now we iterate over the map, construct the tagging index ranges and store them in the db. - for (const [txHashStr, indexes] of txIndexesMap.entries()) { + // Now we iterate over the map, reconstruct the preTags and tx hash and store them in the db. + for (const [txHashStr, highestIndex] of highestIndexMap.entries()) { const txHash = TxHash.fromString(txHashStr); - const ranges = [{ extendedSecret, lowestIndex: Math.min(...indexes), highestIndex: Math.max(...indexes) }]; - await taggingStore.storePendingIndexes(ranges, txHash, jobId); + await taggingStore.storePendingIndexes([{ extendedSecret, index: highestIndex }], txHash, jobId); } } @@ -58,28 +56,20 @@ async function getTxsContainingTags( return allLogs.map(logs => logs.map(log => log.txHash)); } -// Returns a map of txHash to all indexes for that txHash. -function getTxIndexesMap(txHashesForTags: TxHash[][], start: number, count: number): Map { +// Returns a map of txHash to the highest index for that txHash. +function getTxHighestIndexMap(txHashesForTags: TxHash[][], start: number, count: number): Map { if (txHashesForTags.length !== count) { throw new Error(`Number of tx hashes arrays does not match number of tags. ${txHashesForTags.length} !== ${count}`); } - const indexesMap = new Map(); - // Iterate over indexes + const highestIndexMap = new Map(); for (let i = 0; i < txHashesForTags.length; i++) { const taggingIndex = start + i; const txHashesForTag = txHashesForTags[i]; - // iterate over tx hashes that used that index (tag) for (const txHash of txHashesForTag) { const key = txHash.toString(); - const existing = indexesMap.get(key); - // Add the index to the tx's indexes - if (existing) { - existing.push(taggingIndex); - } else { - indexesMap.set(key, [taggingIndex]); - } + highestIndexMap.set(key, Math.max(highestIndexMap.get(key) ?? 0, taggingIndex)); } } - return indexesMap; + return highestIndexMap; } diff --git a/yarn-project/stdlib/src/logs/index.ts b/yarn-project/stdlib/src/logs/index.ts index 540d1fe99698..2e25c40da7c3 100644 --- a/yarn-project/stdlib/src/logs/index.ts +++ b/yarn-project/stdlib/src/logs/index.ts @@ -1,6 +1,5 @@ export * from './extended_directional_app_tagging_secret.js'; export * from './pre_tag.js'; -export * from './tagging_index_range.js'; export * from './contract_class_log.js'; export * from './public_log.js'; export * from './private_log.js'; diff --git a/yarn-project/stdlib/src/logs/tagging_index_range.ts b/yarn-project/stdlib/src/logs/tagging_index_range.ts deleted file mode 100644 index 6392ac8fd26a..000000000000 --- a/yarn-project/stdlib/src/logs/tagging_index_range.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { schemas } from '@aztec/foundation/schemas'; - -import { z } from 'zod'; - -import { - type ExtendedDirectionalAppTaggingSecret, - ExtendedDirectionalAppTaggingSecretSchema, -} from './extended_directional_app_tagging_secret.js'; - -/** - * Represents a range of tagging indexes for a given extended directional app tagging secret. Used to track the lowest - * and highest indexes used in a transaction for a given (sender, recipient, app/contract) tuple. - */ -export type TaggingIndexRange = { - extendedSecret: ExtendedDirectionalAppTaggingSecret; - lowestIndex: number; - highestIndex: number; -}; - -export const TaggingIndexRangeSchema = z.object({ - extendedSecret: ExtendedDirectionalAppTaggingSecretSchema, - lowestIndex: schemas.Integer, - highestIndex: schemas.Integer, -}); diff --git a/yarn-project/stdlib/src/tx/private_execution_result.ts b/yarn-project/stdlib/src/tx/private_execution_result.ts index 4432901b8901..4ddd06352e08 100644 --- a/yarn-project/stdlib/src/tx/private_execution_result.ts +++ b/yarn-project/stdlib/src/tx/private_execution_result.ts @@ -11,7 +11,7 @@ import { PrivateCircuitPublicInputs } from '../kernel/private_circuit_public_inp import type { IsEmpty } from '../kernel/utils/interfaces.js'; import { sortByCounter } from '../kernel/utils/order_and_comparison.js'; import { ContractClassLog, ContractClassLogFields } from '../logs/contract_class_log.js'; -import { type TaggingIndexRange, TaggingIndexRangeSchema } from '../logs/tagging_index_range.js'; +import { type PreTag, PreTagSchema } from '../logs/pre_tag.js'; import { Note } from '../note/note.js'; import { type ZodFor, mapSchema, schemas } from '../schemas/index.js'; import { HashedValues } from './hashed_values.js'; @@ -137,8 +137,8 @@ export class PrivateCallExecutionResult { public returnValues: Fr[], /** The offchain effects emitted during execution of this function call via the `emit_offchain_effect` oracle. */ public offchainEffects: { data: Fr[] }[], - /** The tagging index ranges used in this tx to compute tags for private logs */ - public taggingIndexRanges: TaggingIndexRange[], + /** The pre-tags used in this tx to compute tags for private logs */ + public preTags: PreTag[], /** The nested executions. */ public nestedExecutionResults: PrivateCallExecutionResult[], /** @@ -161,7 +161,7 @@ export class PrivateCallExecutionResult { noteHashNullifierCounterMap: mapSchema(z.coerce.number(), z.number()), returnValues: z.array(schemas.Fr), offchainEffects: z.array(z.object({ data: z.array(schemas.Fr) })), - taggingIndexRanges: z.array(TaggingIndexRangeSchema), + preTags: z.array(PreTagSchema), nestedExecutionResults: z.array(z.lazy(() => PrivateCallExecutionResult.schema)), contractClassLogs: z.array(CountedContractClassLog.schema), }) @@ -178,7 +178,7 @@ export class PrivateCallExecutionResult { fields.noteHashNullifierCounterMap, fields.returnValues, fields.offchainEffects, - fields.taggingIndexRanges, + fields.preTags, fields.nestedExecutionResults, fields.contractClassLogs, );