diff --git a/yarn-project/pxe/src/contract_function_simulator/execution_tagging_index_cache.ts b/yarn-project/pxe/src/contract_function_simulator/execution_tagging_index_cache.ts index 37ffc83016d9..27612bf8ceaf 100644 --- a/yarn-project/pxe/src/contract_function_simulator/execution_tagging_index_cache.ts +++ b/yarn-project/pxe/src/contract_function_simulator/execution_tagging_index_cache.ts @@ -1,32 +1,37 @@ -import { ExtendedDirectionalAppTaggingSecret, type PreTag } from '@aztec/stdlib/logs'; +import { ExtendedDirectionalAppTaggingSecret, type TaggingIndexRange } from '@aztec/stdlib/logs'; /** - * A map that stores the tagging index for a given extended directional app tagging secret. + * A map that stores the tagging index range for a given extended directional app tagging secret. * Note: The directional app tagging secret is unique for a (sender, recipient, contract) tuple while the direction * of sender -> recipient matters. */ export class ExecutionTaggingIndexCache { - private taggingIndexMap: Map = new Map(); + private taggingIndexMap: Map = new Map(); public getLastUsedIndex(secret: ExtendedDirectionalAppTaggingSecret): number | undefined { - return this.taggingIndexMap.get(secret.toString()); + return this.taggingIndexMap.get(secret.toString())?.highestIndex; } public setLastUsedIndex(secret: ExtendedDirectionalAppTaggingSecret, index: number) { const currentValue = this.taggingIndexMap.get(secret.toString()); - if (currentValue !== undefined && currentValue !== index - 1) { - throw new Error(`Invalid tagging index update. Current value: ${currentValue}, new value: ${index}`); + if (currentValue !== undefined && currentValue.highestIndex !== index - 1) { + throw new Error(`Invalid tagging index update. Current value: ${currentValue.highestIndex}, new value: ${index}`); + } + if (currentValue !== undefined) { + currentValue.highestIndex = index; + } else { + this.taggingIndexMap.set(secret.toString(), { lowestIndex: index, highestIndex: index }); } - this.taggingIndexMap.set(secret.toString(), index); } /** - * Returns the pre-tags that were used in this execution (and that need to be stored in the db). + * Returns the tagging index ranges that were used in this execution (and that need to be stored in the db). */ - public getUsedPreTags(): PreTag[] { - return Array.from(this.taggingIndexMap.entries()).map(([secret, index]) => ({ + public getUsedTaggingIndexRanges(): TaggingIndexRange[] { + return Array.from(this.taggingIndexMap.entries()).map(([secret, { lowestIndex, highestIndex }]) => ({ extendedSecret: ExtendedDirectionalAppTaggingSecret.fromString(secret), - index, + lowestIndex, + highestIndex, })); } } diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.ts index 5dbbd6509a9b..5923f05510e1 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.ts @@ -81,7 +81,7 @@ export async function executePrivateFunction( const newNotes = privateExecutionOracle.getNewNotes(); const noteHashNullifierCounterMap = privateExecutionOracle.getNoteHashNullifierCounterMap(); const offchainEffects = privateExecutionOracle.getOffchainEffects(); - const preTags = privateExecutionOracle.getUsedPreTags(); + const taggingIndexRanges = privateExecutionOracle.getUsedTaggingIndexRanges(); const nestedExecutionResults = privateExecutionOracle.getNestedExecutionResults(); let timerSubtractionList = nestedExecutionResults; @@ -104,7 +104,7 @@ export async function executePrivateFunction( noteHashNullifierCounterMap, rawReturnValues, offchainEffects, - preTags, + taggingIndexRanges, nestedExecutionResults, contractClassLogs, { diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts index 6c4a9914c8f7..e0b8748e7409 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts @@ -14,7 +14,7 @@ import { import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { siloNullifier } from '@aztec/stdlib/hash'; import { PrivateContextInputs } from '@aztec/stdlib/kernel'; -import { type ContractClassLog, ExtendedDirectionalAppTaggingSecret, type PreTag } from '@aztec/stdlib/logs'; +import { type ContractClassLog, ExtendedDirectionalAppTaggingSecret, type TaggingIndexRange } from '@aztec/stdlib/logs'; import { Tag } from '@aztec/stdlib/logs'; import { Note, type NoteStatus } from '@aztec/stdlib/note'; import { @@ -166,10 +166,10 @@ export class PrivateExecutionOracle extends UtilityExecutionOracle implements IP } /** - * Returns the pre-tags that were used in this execution (and that need to be stored in the db). + * Returns the tagging index ranges that were used in this execution (and that need to be stored in the db). */ - public getUsedPreTags(): PreTag[] { - return this.taggingIndexCache.getUsedPreTags(); + public getUsedTaggingIndexRanges(): TaggingIndexRange[] { + return this.taggingIndexCache.getUsedTaggingIndexRanges(); } /** diff --git a/yarn-project/pxe/src/pxe.ts b/yarn-project/pxe/src/pxe.ts index ffdb6b1c81b8..11653b08d834 100644 --- a/yarn-project/pxe/src/pxe.ts +++ b/yarn-project/pxe/src/pxe.ts @@ -766,17 +766,17 @@ export class PXE { // transaction before this one is included in a block from this PXE, and that transaction contains a log with // a tag derived from the same secret, we would reuse the tag and the transactions would be linked. Hence // storing the tags here prevents linkage of txs sent from the same PXE. - const preTagsUsedInTheTx = privateExecutionResult.entrypoint.preTags; - if (preTagsUsedInTheTx.length > 0) { + const taggingIndexRangesUsedInTheTx = privateExecutionResult.entrypoint.taggingIndexRanges; + if (taggingIndexRangesUsedInTheTx.length > 0) { // TODO(benesjan): The following is an expensive operation. Figure out a way to avoid it. const txHash = (await txProvingResult.toTx()).txHash; - await this.senderTaggingStore.storePendingIndexes(preTagsUsedInTheTx, txHash, jobId); - this.log.debug(`Stored used pre-tags as sender for the tx`, { - preTagsUsedInTheTx, + await this.senderTaggingStore.storePendingIndexes(taggingIndexRangesUsedInTheTx, txHash, jobId); + this.log.debug(`Stored used tagging index ranges as sender for the tx`, { + taggingIndexRangesUsedInTheTx, }); } else { - this.log.debug(`No pre-tags used in the tx`); + this.log.debug(`No tagging index ranges used in the tx`); } return txProvingResult; diff --git a/yarn-project/pxe/src/storage/metadata.ts b/yarn-project/pxe/src/storage/metadata.ts index cb1dee391377..826f90735b91 100644 --- a/yarn-project/pxe/src/storage/metadata.ts +++ b/yarn-project/pxe/src/storage/metadata.ts @@ -1 +1 @@ -export const PXE_DATA_SCHEMA_VERSION = 3; +export const PXE_DATA_SCHEMA_VERSION = 4; diff --git a/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts b/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts index 986f1daef6fc..b2800582f02d 100644 --- a/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts +++ b/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts @@ -1,11 +1,19 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; -import type { ExtendedDirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs'; +import { RevertCode } from '@aztec/stdlib/avm'; +import type { ExtendedDirectionalAppTaggingSecret, TaggingIndexRange } from '@aztec/stdlib/logs'; +import { PrivateLog, SiloedTag } from '@aztec/stdlib/logs'; import { randomExtendedDirectionalAppTaggingSecret } from '@aztec/stdlib/testing'; -import { TxHash } from '@aztec/stdlib/tx'; +import { TxEffect, TxHash } from '@aztec/stdlib/tx'; import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/constants.js'; import { SenderTaggingStore } from './sender_tagging_store.js'; +/** Helper to create a single-index range (lowestIndex === highestIndex). */ +function range(secret: ExtendedDirectionalAppTaggingSecret, lowest: number, highest?: number): TaggingIndexRange { + return { extendedSecret: secret, lowestIndex: lowest, highestIndex: highest ?? lowest }; +} + describe('SenderTaggingStore', () => { let taggingStore: SenderTaggingStore; let secret1: ExtendedDirectionalAppTaggingSecret; @@ -18,25 +26,20 @@ describe('SenderTaggingStore', () => { }); describe('storePendingIndexes', () => { - it('stores a single pending index', async () => { + it('stores a single pending index range', async () => { const txHash = TxHash.random(); - const preTag: PreTag = { extendedSecret: secret1, index: 5 }; - await taggingStore.storePendingIndexes([preTag], txHash, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); expect(txHashes).toHaveLength(1); expect(txHashes[0]).toEqual(txHash); }); - it('stores multiple pending indexes for different secrets', async () => { + it('stores multiple pending index ranges for different secrets', async () => { const txHash = TxHash.random(); - const preTags: PreTag[] = [ - { extendedSecret: secret1, index: 3 }, - { extendedSecret: secret2, index: 7 }, - ]; - await taggingStore.storePendingIndexes(preTags, txHash, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3), range(secret2, 7)], txHash, 'test'); const txHashes1 = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); expect(txHashes1).toHaveLength(1); @@ -47,12 +50,12 @@ describe('SenderTaggingStore', () => { expect(txHashes2[0]).toEqual(txHash); }); - it('stores multiple pending indexes for the same secret from different txs', async () => { + it('stores multiple pending index ranges for the same secret from different txs', async () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); expect(txHashes).toHaveLength(2); @@ -60,68 +63,71 @@ describe('SenderTaggingStore', () => { expect(txHashes).toContainEqual(txHash2); }); - it('ignores duplicate preTag + txHash combination', async () => { + it('ignores duplicate range + txHash combination', async () => { const txHash = TxHash.random(); - const preTag: PreTag = { extendedSecret: secret1, index: 5 }; - await taggingStore.storePendingIndexes([preTag], txHash, 'test'); - await taggingStore.storePendingIndexes([preTag], txHash, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); expect(txHashes).toHaveLength(1); expect(txHashes[0]).toEqual(txHash); }); - it('throws when storing duplicate secrets in the same call', async () => { + it('stores a range spanning multiple indexes', async () => { const txHash = TxHash.random(); - const preTags: PreTag[] = [ - { extendedSecret: secret1, index: 3 }, - { extendedSecret: secret1, index: 7 }, - ]; - await expect(taggingStore.storePendingIndexes(preTags, txHash, 'test')).rejects.toThrow( - 'Duplicate secrets found when storing pending indexes', - ); + await taggingStore.storePendingIndexes([range(secret1, 3, 7)], txHash, 'test'); + + // By design the txs are filtered based on the highestIndex (7) in getTxHashesOfPendingIndexes so we shouldn't + // receive the tx only in the second query. + const txHashesNotContainingHighest = await taggingStore.getTxHashesOfPendingIndexes(secret1, 3, 4, 'test'); + expect(txHashesNotContainingHighest).toHaveLength(0); + + const txHashesContainingHighest = await taggingStore.getTxHashesOfPendingIndexes(secret1, 7, 8, 'test'); + expect(txHashesContainingHighest).toHaveLength(1); + expect(txHashesContainingHighest[0]).toEqual(txHash); + + expect(await taggingStore.getLastUsedIndex(secret1, 'test')).toBe(7); }); - it('throws when storing a different index for an existing secret + txHash pair', async () => { + it('throws when storing a different range for an existing secret + txHash pair', async () => { const txHash = TxHash.random(); - // First store an index - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); - // Try to store a different index for the same secret + txHash pair - await expect( - taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash, 'test'), - ).rejects.toThrow(/Cannot store index 7.*a different index 5 already exists/); + // Storing a different range for the same secret + txHash should throw + await expect(taggingStore.storePendingIndexes([range(secret1, 7)], txHash, 'test')).rejects.toThrow( + /Conflicting range/, + ); }); - it('throws when storing a pending index lower than the last finalized index', async () => { + it('throws when storing a pending index range lower than the last finalized index', async () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); // First store and finalize an index - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 10 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 10)], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Try to store a pending index lower than the finalized index - await expect( - taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash2, 'test'), - ).rejects.toThrow(/Cannot store pending index 5.*lower than or equal to the last finalized index 10/); + await expect(taggingStore.storePendingIndexes([range(secret1, 5)], txHash2, 'test')).rejects.toThrow( + /lowestIndex is lower than or equal to the last finalized index 10/, + ); }); - it('throws when storing a pending index equal to the last finalized index', async () => { + it('throws when storing a pending index range equal to the last finalized index', async () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); // First store and finalize an index - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 10 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 10)], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Try to store a pending index equal to the finalized index - await expect( - taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 10 }], txHash2, 'test'), - ).rejects.toThrow(/Cannot store pending index 10.*lower than or equal to the last finalized index 10/); + await expect(taggingStore.storePendingIndexes([range(secret1, 10)], txHash2, 'test')).rejects.toThrow( + /lowestIndex is lower than or equal to the last finalized index 10/, + ); }); it('allows storing a pending index higher than the last finalized index', async () => { @@ -129,13 +135,11 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); // First store and finalize an index - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 10 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 10)], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Store a pending index higher than the finalized index - should succeed - await expect( - taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 15 }], txHash2, 'test'), - ).resolves.not.toThrow(); + await expect(taggingStore.storePendingIndexes([range(secret1, 15)], txHash2, 'test')).resolves.not.toThrow(); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 20, 'test'); expect(txHashes).toHaveLength(1); @@ -150,12 +154,12 @@ describe('SenderTaggingStore', () => { const indexBeyondWindow = finalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; // First store and finalize an index - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: finalizedIndex }], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, finalizedIndex)], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Try to store an index beyond the window await expect( - taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: indexBeyondWindow }], txHash2, 'test'), + taggingStore.storePendingIndexes([range(secret1, indexBeyondWindow)], txHash2, 'test'), ).rejects.toThrow( `Highest used index ${indexBeyondWindow} is further than window length from the highest finalized index ${finalizedIndex}`, ); @@ -168,12 +172,12 @@ describe('SenderTaggingStore', () => { const indexAtBoundary = finalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; // First store and finalize an index - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: finalizedIndex }], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, finalizedIndex)], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Store an index at the boundary, but check is >, so it should succeed await expect( - taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: indexAtBoundary }], txHash2, 'test'), + taggingStore.storePendingIndexes([range(secret1, indexAtBoundary)], txHash2, 'test'), ).resolves.not.toThrow(); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, indexAtBoundary + 5, 'test'); @@ -194,9 +198,9 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); const txHash3 = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash2, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 8 }], txHash3, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 8)], txHash3, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 4, 9, 'test'); expect(txHashes).toHaveLength(2); @@ -209,8 +213,8 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash1, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 10 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 10)], txHash2, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 5, 10, 'test'); expect(txHashes).toHaveLength(1); @@ -223,16 +227,16 @@ describe('SenderTaggingStore', () => { const txHash3 = TxHash.random(); const txHash4 = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash2, 'test'); // We store different secret with txHash1 to check we correctly don't return it in the result - await taggingStore.storePendingIndexes([{ extendedSecret: secret2, index: 7 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret2, 7)], txHash1, 'test'); // Store "parallel" index for secret1 with a different tx (can happen when sending logs from multiple PXEs) - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash3, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash4, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash3, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash4, 'test'); const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); - // Should have 3 unique tx hashes for secret1 + // Should have 4 unique tx hashes for secret1 expect(txHashes).toEqual(expect.arrayContaining([txHash1, txHash2, txHash3, txHash4])); }); }); @@ -245,7 +249,7 @@ describe('SenderTaggingStore', () => { it('returns the last finalized index after finalizePendingIndexes', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); await taggingStore.finalizePendingIndexes([txHash], 'test'); const lastFinalized = await taggingStore.getLastFinalizedIndex(secret1, 'test'); @@ -261,7 +265,7 @@ describe('SenderTaggingStore', () => { it('returns the last finalized index when no pending indexes exist', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); await taggingStore.finalizePendingIndexes([txHash], 'test'); const lastUsed = await taggingStore.getLastUsedIndex(secret1, 'test'); @@ -273,11 +277,11 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); // First, finalize an index - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Then add a higher pending index - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); const lastUsed = await taggingStore.getLastUsedIndex(secret1, 'test'); expect(lastUsed).toBe(7); @@ -288,9 +292,9 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); const txHash3 = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash3, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash3, 'test'); const lastUsed = await taggingStore.getLastUsedIndex(secret1, 'test'); expect(lastUsed).toBe(7); @@ -302,9 +306,9 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret2, index: 5 }], txHash1, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret2, 5)], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); await taggingStore.dropPendingIndexes([txHash1], 'test'); @@ -322,7 +326,7 @@ describe('SenderTaggingStore', () => { describe('finalizePendingIndexes', () => { it('moves pending index to finalized for a given tx hash', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash, 'test'); await taggingStore.finalizePendingIndexes([txHash], 'test'); @@ -338,10 +342,10 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); await taggingStore.finalizePendingIndexes([txHash2], 'test'); const lastFinalized = await taggingStore.getLastFinalizedIndex(secret1, 'test'); @@ -353,8 +357,8 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); // Store both pending indexes first - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash1, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash2, 'test'); // Finalize the higher index first await taggingStore.finalizePendingIndexes([txHash1], 'test'); @@ -366,14 +370,14 @@ describe('SenderTaggingStore', () => { expect(lastFinalized).toBe(7); // Should remain at 7 }); - it('prunes pending indexes with lower or equal index than finalized', async () => { + it('prunes pending indexes with lower or equal highestIndex than finalized', async () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); const txHash3 = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash2, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash3, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash3, 'test'); // Finalize txHash2 (index 5) await taggingStore.finalizePendingIndexes([txHash2], 'test'); @@ -387,14 +391,7 @@ describe('SenderTaggingStore', () => { it('handles multiple secrets in the same tx', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes( - [ - { extendedSecret: secret1, index: 3 }, - { extendedSecret: secret2, index: 7 }, - ], - txHash, - 'test', - ); + await taggingStore.storePendingIndexes([range(secret1, 3), range(secret2, 7)], txHash, 'test'); await taggingStore.finalizePendingIndexes([txHash], 'test'); @@ -405,9 +402,19 @@ describe('SenderTaggingStore', () => { expect(lastFinalized2).toBe(7); }); + it('finalizes the highestIndex of a range', async () => { + const txHash = TxHash.random(); + await taggingStore.storePendingIndexes([range(secret1, 3, 7)], txHash, 'test'); + + await taggingStore.finalizePendingIndexes([txHash], 'test'); + + const lastFinalized = await taggingStore.getLastFinalizedIndex(secret1, 'test'); + expect(lastFinalized).toBe(7); + }); + it('does nothing when tx hash does not exist', async () => { const txHash = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash, 'test'); await taggingStore.finalizePendingIndexes([TxHash.random()], 'test'); @@ -427,7 +434,7 @@ describe('SenderTaggingStore', () => { const txHash2 = TxHash.random(); // Step 1: Add pending index - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); expect(await taggingStore.getLastUsedIndex(secret1, 'test')).toBe(3); expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBeUndefined(); @@ -437,7 +444,7 @@ describe('SenderTaggingStore', () => { expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(3); // Step 3: Add a new higher pending index - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, 'test'); expect(await taggingStore.getLastUsedIndex(secret1, 'test')).toBe(7); expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(3); @@ -451,8 +458,8 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 5)], txHash2, 'test'); expect(await taggingStore.getLastUsedIndex(secret1, 'test')).toBe(5); @@ -468,14 +475,14 @@ describe('SenderTaggingStore', () => { const txHash3 = TxHash.random(); // Secret1: pending -> finalized - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, 'test'); await taggingStore.finalizePendingIndexes([txHash1], 'test'); // Secret2: pending (not finalized) - await taggingStore.storePendingIndexes([{ extendedSecret: secret2, index: 5 }], txHash2, 'test'); + await taggingStore.storePendingIndexes([range(secret2, 5)], txHash2, 'test'); // Secret1: new pending - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash3, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash3, 'test'); expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(3); expect(await taggingStore.getLastUsedIndex(secret1, 'test')).toBe(7); @@ -484,18 +491,135 @@ describe('SenderTaggingStore', () => { }); }); + describe('finalizePendingIndexesOfAPartiallyRevertedTx', () => { + function makeTxEffect(txHash: TxHash, siloedTags: SiloedTag[]): TxEffect { + return new TxEffect( + RevertCode.APP_LOGIC_REVERTED, + txHash, + Fr.ZERO, + [Fr.random()], // noteHashes (at least 1 nullifier required below, not here) + [Fr.random()], // nullifiers (at least 1 required) + [], // l2ToL1Msgs + [], // publicDataWrites + siloedTags.map(tag => PrivateLog.random(tag.value)), // privateLogs with surviving tags + [], // publicLogs + [], // contractClassLogs + ); + } + + it('finalizes only the indexes whose tags appear in TxEffect', async () => { + const txHash = TxHash.random(); + + // Store a range [3, 5] for secret1 in the same tx + await taggingStore.storePendingIndexes([range(secret1, 3, 5)], txHash, 'test'); + + // Compute the siloed tag for index 3 (the one that survives) + const survivingTag = await SiloedTag.compute({ extendedSecret: secret1, index: 3 }); + const txEffect = makeTxEffect(txHash, [survivingTag]); + + await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(txEffect, 'test'); + + // Index 3 should be finalized (it was onchain) + expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(3); + // All pending indexes for this tx should be removed + const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); + expect(txHashes).toHaveLength(0); + }); + + it('drops all indexes when no tags survive onchain', async () => { + const txHash = TxHash.random(); + + await taggingStore.storePendingIndexes([range(secret1, 3, 5)], txHash, 'test'); + + // TxEffect with no matching private logs (empty) + const txEffect = makeTxEffect(txHash, []); + + await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(txEffect, 'test'); + + // No finalized index should be set + expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBeUndefined(); + // All pending indexes for this tx should be removed + const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); + expect(txHashes).toHaveLength(0); + }); + + it('handles multiple secrets affected by the same partially reverted tx', async () => { + const txHash = TxHash.random(); + + // Store pending index ranges for both secrets in the same tx + await taggingStore.storePendingIndexes([range(secret1, 3, 5), range(secret2, 7)], txHash, 'test'); + + // Only index 3 for secret1 survives onchain; other indexes for secret1 and secret2 are dropped + const survivingTag = await SiloedTag.compute({ extendedSecret: secret1, index: 3 }); + const txEffect = makeTxEffect(txHash, [survivingTag]); + + await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(txEffect, 'test'); + + // secret1: index 3 should be finalized + expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(3); + expect(await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test')).toHaveLength(0); + + // secret2: no finalized index, all pending removed + expect(await taggingStore.getLastFinalizedIndex(secret2, 'test')).toBeUndefined(); + expect(await taggingStore.getTxHashesOfPendingIndexes(secret2, 0, 10, 'test')).toHaveLength(0); + }); + + it('preserves pending indexes from other txs', async () => { + const revertedTxHash = TxHash.random(); + const otherTxHash = TxHash.random(); + + // Store pending indexes: one from reverted tx, one from another tx + await taggingStore.storePendingIndexes([range(secret1, 3)], revertedTxHash, 'test'); + await taggingStore.storePendingIndexes([range(secret1, 7)], otherTxHash, 'test'); + + // TxEffect with no surviving tags for the reverted tx + const txEffect = makeTxEffect(revertedTxHash, []); + + await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(txEffect, 'test'); + + // No finalized index (nothing survived from the reverted tx) + expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBeUndefined(); + // The other tx's pending index should still be there + const txHashes = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test'); + expect(txHashes).toHaveLength(1); + expect(txHashes[0]).toEqual(otherTxHash); + }); + + it('correctly updates finalized index when there is an existing finalized index', async () => { + const txHash1 = TxHash.random(); + const revertedTxHash = TxHash.random(); + + // Store and finalize index 2 + await taggingStore.storePendingIndexes([range(secret1, 2)], txHash1, 'test'); + await taggingStore.finalizePendingIndexes([txHash1], 'test'); + + // Store a pending range [4, 6] for a partially reverted tx + await taggingStore.storePendingIndexes([range(secret1, 4, 6)], revertedTxHash, 'test'); + + // Only index 4 survives + const survivingTag = await SiloedTag.compute({ extendedSecret: secret1, index: 4 }); + const txEffect = makeTxEffect(revertedTxHash, [survivingTag]); + + await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(txEffect, 'test'); + + // Finalized index should be updated to 4 (higher than previous 2) + expect(await taggingStore.getLastFinalizedIndex(secret1, 'test')).toBe(4); + expect(await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'test')).toHaveLength(0); + }); + }); + describe('staged writes', () => { it('writes of uncommitted jobs are not visible outside the job that makes them', async () => { const committedTxHash = TxHash.random(); { const commitJobId: string = 'commit-job'; - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], committedTxHash, commitJobId); + await taggingStore.storePendingIndexes([range(secret1, 3)], committedTxHash, commitJobId); await taggingStore.commit(commitJobId); } const stagedTxHash = TxHash.random(); const stagingJobId: string = 'staging-job'; - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 5 }], stagedTxHash, stagingJobId); + await taggingStore.storePendingIndexes([range(secret1, 5)], stagedTxHash, stagingJobId); // For a job without any staged data we should only get committed data const txHashesWithoutJobId = await taggingStore.getTxHashesOfPendingIndexes(secret1, 0, 10, 'no-data-job'); @@ -513,7 +637,7 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); { const commitJobId: string = 'commit-job'; - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash1, commitJobId); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash1, commitJobId); await taggingStore.finalizePendingIndexes([txHash1], commitJobId); await taggingStore.commit(commitJobId); } @@ -522,7 +646,7 @@ describe('SenderTaggingStore', () => { const stagingJobId: string = 'staging-job'; // Stage a higher finalized index (not committed) - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash2, stagingJobId); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash2, stagingJobId); await taggingStore.finalizePendingIndexes([txHash2], stagingJobId); // With a different jobId, should get the committed finalized index @@ -537,8 +661,8 @@ describe('SenderTaggingStore', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); const commitJobId: string = 'commit-job'; - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 2 }], txHash1, commitJobId); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 3 }], txHash2, commitJobId); + await taggingStore.storePendingIndexes([range(secret1, 2)], txHash1, commitJobId); + await taggingStore.storePendingIndexes([range(secret1, 3)], txHash2, commitJobId); await taggingStore.finalizePendingIndexes([txHash1], commitJobId); await taggingStore.commit(commitJobId); } @@ -546,7 +670,7 @@ describe('SenderTaggingStore', () => { const stagingJobId: string = 'staging-job'; { const txHash3 = TxHash.random(); - await taggingStore.storePendingIndexes([{ extendedSecret: secret1, index: 7 }], txHash3, stagingJobId); + await taggingStore.storePendingIndexes([range(secret1, 7)], txHash3, stagingJobId); await taggingStore.finalizePendingIndexes([txHash3], stagingJobId); await taggingStore.discardStaged(stagingJobId); } diff --git a/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.ts b/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.ts index 1b15bbbb207a..05f79be89b88 100644 --- a/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.ts +++ b/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.ts @@ -1,10 +1,13 @@ import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store'; -import type { ExtendedDirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs'; -import { TxHash } from '@aztec/stdlib/tx'; +import { ExtendedDirectionalAppTaggingSecret, SiloedTag, type TaggingIndexRange } from '@aztec/stdlib/logs'; +import { TxEffect, TxHash } from '@aztec/stdlib/tx'; import type { StagedStore } from '../../job_coordinator/job_coordinator.js'; import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/constants.js'; +/** Internal representation of a pending index range entry. */ +type PendingIndexesEntry = { lowestIndex: number; highestIndex: number; txHash: string }; + /** * Data provider of tagging data used when syncing the sender tagging indexes. The recipient counterpart of this class * is called RecipientTaggingStore. We have the data stores separate for sender and recipient because @@ -15,20 +18,19 @@ export class SenderTaggingStore implements StagedStore { #store: AztecAsyncKVStore; - // Stores the pending indexes for each directional app tagging secret. Pending here means that the tx that contained - // the private logs with tags corresponding to these indexes has not been finalized yet. - // - // We don't store just the highest index because if their transaction is dropped we'd then need the information about - // the lower pending indexes. For each secret-tx pair however we only store the largest index used in that tx, since - // the smaller ones are irrelevant due to tx atomicity. + // Stores the pending index ranges for each directional app tagging secret. Pending here means that the tx that + // contained the private logs with tags corresponding to these indexes has not been finalized yet. // - // TODO(#17615): This assumes no logs are used in the non-revertible phase. + // We store the full range (lowestIndex, highestIndex) for each secret-tx pair because transactions can partially + // revert, in which case only some logs (from the non-revertible phase) survive onchain. By storing the range, + // we can expand it and check each individual siloed tag against the TxEffect to determine which indexes made it + // onchain. // - // directional app tagging secret => { pending index, txHash }[] - #pendingIndexes: AztecAsyncMap; + // directional app tagging secret => { lowestIndex, highestIndex, txHash }[] + #pendingIndexes: AztecAsyncMap; - // jobId => directional app tagging secret => { pending index, txHash }[] - #pendingIndexesForJob: Map>; + // jobId => directional app tagging secret => { lowestIndex, highestIndex, txHash }[] + #pendingIndexesForJob: Map>; // Stores the last (highest) finalized index for each directional app tagging secret. We care only about the last // index because unlike the pending indexes, it will never happen that a finalized index would be removed and hence @@ -50,7 +52,7 @@ export class SenderTaggingStore implements StagedStore { this.#lastFinalizedIndexesForJob = new Map(); } - #getPendingIndexesForJob(jobId: string): Map { + #getPendingIndexesForJob(jobId: string): Map { let pendingIndexesForJob = this.#pendingIndexesForJob.get(jobId); if (!pendingIndexesForJob) { pendingIndexesForJob = new Map(); @@ -68,7 +70,7 @@ export class SenderTaggingStore implements StagedStore { return jobStagedLastFinalizedIndexes; } - async #readPendingIndexes(jobId: string, secret: string): Promise<{ index: number; txHash: string }[]> { + async #readPendingIndexes(jobId: string, secret: string): Promise { // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there // are no pending read requests). The staged value still takes precedence if it exists. const dbValue = await this.#pendingIndexes.getAsync(secret); @@ -76,7 +78,7 @@ export class SenderTaggingStore implements StagedStore { return staged !== undefined ? staged : (dbValue ?? []); } - #writePendingIndexes(jobId: string, secret: string, pendingIndexes: { index: number; txHash: string }[]) { + #writePendingIndexes(jobId: string, secret: string, pendingIndexes: PendingIndexesEntry[]) { this.#getPendingIndexesForJob(jobId).set(secret, pendingIndexes); } @@ -126,57 +128,37 @@ export class SenderTaggingStore implements StagedStore { } /** - * Stores pending indexes. - * @remarks Ignores the index if the same preTag + txHash combination already exists in the db with the same index. - * This is expected to happen because whenever we start sync we start from the last finalized index and we can have - * pending indexes already stored from previous syncs. - * @param preTags - The pre-tags containing the directional app tagging secrets and the indexes that are to be - * stored in the db. - * @param txHash - The tx in which the pretags were used in private logs. + * Stores pending index ranges. + * @remarks If the same (secret, txHash) pair already exists in the db with an equal range, it's a no-op. This is + * expected to happen because whenever we start sync we start from the last finalized index and we can have pending + * ranges already stored from previous syncs. If the ranges differ, it throws an error as that indicates a bug. + * @param ranges - The tagging index ranges containing the directional app tagging secrets and the index ranges that are + * to be stored in the db. + * @param txHash - The tx in which the tagging indexes were used in private logs. * @param jobId - job context for staged writes to this store. See `JobCoordinator` for more details. - * @throws If any two pre-tags contain the same directional app tagging secret. This is enforced because we care - * only about the highest index for a given secret that was used in the tx. Hence this check is a good way to catch - * bugs. - * @throws If the newly stored pending index is further than window length from the highest finalized index for the - * same secret. This is enforced in order to give a guarantee to a recipient that he doesn't need to look further than - * window length ahead of the highest finalized index. - * @throws If a secret + txHash pair already exists in the db with a different index value. It should never happen - * that we would attempt to store a different index for a given secret-txHash pair because we always store just the - * highest index for a given secret-txHash pair. Hence this is a good way to catch bugs. - * @throws If the newly stored pending index is lower than or equal to the last finalized index for the same secret. - * This is enforced because this should never happen if the syncing is done correctly as we look for logs from higher - * indexes than finalized ones. + * @throws If the highestIndex is further than window length from the highest finalized index for the same secret. + * @throws If the lowestIndex is lower than or equal to the last finalized index for the same secret. + * @throws If a different range already exists for the same (secret, txHash) pair. */ - storePendingIndexes(preTags: PreTag[], txHash: TxHash, jobId: string): Promise { - if (preTags.length === 0) { + storePendingIndexes(ranges: TaggingIndexRange[], txHash: TxHash, jobId: string): Promise { + if (ranges.length === 0) { return Promise.resolve(); } - // The secrets in pre-tags should be unique because we always store just the highest index per given secret-txHash - // pair. Below we check that this is the case. - const secretsSet = new Set(preTags.map(preTag => preTag.extendedSecret.toString())); - if (secretsSet.size !== preTags.length) { - return Promise.reject(new Error(`Duplicate secrets found when storing pending indexes`)); - } - const txHashStr = txHash.toString(); return this.#store.transactionAsync(async () => { // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive - const preTagReadPromises = preTags.map(({ extendedSecret, index }) => { - const secretStr = extendedSecret.toString(); - return { - extendedSecret, - secretStr, - index, - pending: this.#readPendingIndexes(jobId, secretStr), - finalized: this.#readLastFinalizedIndex(jobId, secretStr), - }; - }); + const rangeReadPromises = ranges.map(range => ({ + range, + secretStr: range.extendedSecret.toString(), + pending: this.#readPendingIndexes(jobId, range.extendedSecret.toString()), + finalized: this.#readLastFinalizedIndex(jobId, range.extendedSecret.toString()), + })); // Await all reads together - const preTagData = await Promise.all( - preTagReadPromises.map(async item => ({ + const rangeData = await Promise.all( + rangeReadPromises.map(async item => ({ ...item, pendingData: await item.pending, finalizedIndex: await item.finalized, @@ -184,48 +166,51 @@ export class SenderTaggingStore implements StagedStore { ); // Process in memory and validate - for (const { secretStr, index, pendingData, finalizedIndex } of preTagData) { - // First we check that for any secret the highest used index in tx is not further than window length from - // the highest finalized index. - if (index > (finalizedIndex ?? 0) + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) { + for (const { range, secretStr, pendingData, finalizedIndex } of rangeData) { + // Check that the highest index is not further than window length from the highest finalized index. + if (range.highestIndex > (finalizedIndex ?? 0) + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) { throw new Error( - `Highest used index ${index} is further than window length from the highest finalized index ${finalizedIndex ?? 0}. + `Highest used index ${range.highestIndex} is further than window length from the highest finalized index ${finalizedIndex ?? 0}. Tagging window length ${UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN} is configured too low. Contact the Aztec team to increase it!`, ); } - // Throw if the new pending index is lower than or equal to the last finalized index - if (finalizedIndex !== undefined && index <= finalizedIndex) { + // Throw if the lowest index is lower than or equal to the last finalized index + if (finalizedIndex !== undefined && range.lowestIndex <= finalizedIndex) { throw new Error( - `Cannot store pending index ${index} for secret ${secretStr}: ` + - `it is lower than or equal to the last finalized index ${finalizedIndex}`, + `Cannot store pending index range [${range.lowestIndex}, ${range.highestIndex}] for secret ${secretStr}: ` + + `lowestIndex is lower than or equal to the last finalized index ${finalizedIndex}`, ); } - // Check if this secret + txHash combination already exists - const existingForSecretAndTx = pendingData.find(entry => entry.txHash === txHashStr); + // Check if an entry with the same txHash already exists + const existingEntry = pendingData.find(entry => entry.txHash === txHashStr); - if (existingForSecretAndTx) { - // If it exists with a different index, throw an error - if (existingForSecretAndTx.index !== index) { + if (existingEntry) { + // Assert that the ranges are equal — different ranges for the same (secret, txHash) indicates a bug + if (existingEntry.lowestIndex !== range.lowestIndex || existingEntry.highestIndex !== range.highestIndex) { throw new Error( - `Cannot store index ${index} for secret ${secretStr} and txHash ${txHashStr}: ` + - `a different index ${existingForSecretAndTx.index} already exists for this secret-txHash pair`, + `Conflicting range for secret ${secretStr} and txHash ${txHashStr}: ` + + `existing [${existingEntry.lowestIndex}, ${existingEntry.highestIndex}] vs ` + + `new [${range.lowestIndex}, ${range.highestIndex}]`, ); } - // If it exists with the same index, ignore the update (no-op) + // Exact duplicate — skip } else { - // If it doesn't exist, add it - this.#writePendingIndexes(jobId, secretStr, [...pendingData, { index, txHash: txHashStr }]); + this.#writePendingIndexes(jobId, secretStr, [ + ...pendingData, + { lowestIndex: range.lowestIndex, highestIndex: range.highestIndex, txHash: txHashStr }, + ]); } } }); } /** - * Returns the transaction hashes of all pending transactions that contain indexes within a specified range - * for a given directional app tagging secret. + * Returns the transaction hashes of all pending transactions that contain highest indexes within a specified range + * for a given directional app tagging secret. We check based on the highest indexes only as that is the relevant + * information for the caller of this function. * @param secret - The directional app tagging secret to query pending indexes for. * @param startIndex - The lower bound of the index range (inclusive). * @param endIndex - The upper bound of the index range (exclusive). @@ -241,7 +226,7 @@ export class SenderTaggingStore implements StagedStore { return this.#store.transactionAsync(async () => { const existing = await this.#readPendingIndexes(jobId, secret.toString()); const txHashes = existing - .filter(entry => entry.index >= startIndex && entry.index < endIndex) + .filter(entry => entry.highestIndex >= startIndex && entry.highestIndex < endIndex) .map(entry => entry.txHash); return Array.from(new Set(txHashes)).map(TxHash.fromString); }); @@ -269,16 +254,15 @@ export class SenderTaggingStore implements StagedStore { const pendingPromise = this.#readPendingIndexes(jobId, secretStr); const finalizedPromise = this.#readLastFinalizedIndex(jobId, secretStr); - const [pendingTxScopedIndexes, lastFinalized] = await Promise.all([pendingPromise, finalizedPromise]); - const pendingIndexes = pendingTxScopedIndexes.map(entry => entry.index); + const [pendingEntries, lastFinalized] = await Promise.all([pendingPromise, finalizedPromise]); - if (pendingTxScopedIndexes.length === 0) { + if (pendingEntries.length === 0) { return lastFinalized; } - // As the last used index we return the highest one from the pending indexes. Note that this value will be always - // higher than the last finalized index because we prune lower pending indexes when a tx is finalized. - return Math.max(...pendingIndexes); + // As the last used index we return the highest one from the pending index ranges. Note that this value will be + // always higher than the last finalized index because we prune lower pending indexes when a tx is finalized. + return Math.max(...pendingEntries.map(entry => entry.highestIndex)); }); } @@ -294,7 +278,7 @@ export class SenderTaggingStore implements StagedStore { return this.#store.transactionAsync(async () => { // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive - const secretReadPromises: Map> = new Map(); + const secretReadPromises: Map> = new Map(); for await (const secret of this.#pendingIndexes.keysAsync()) { secretReadPromises.set(secret, this.#readPendingIndexes(jobId, secret)); @@ -330,22 +314,15 @@ export class SenderTaggingStore implements StagedStore { }); } - /** - * Updates pending indexes corresponding to the given transaction hashes to be finalized and prunes any lower pending - * indexes. - */ - finalizePendingIndexes(txHashes: TxHash[], jobId: string): Promise { - if (txHashes.length === 0) { - return Promise.resolve(); - } - - const txHashStrings = new Set(txHashes.map(tx => tx.toString())); - + /** Prefetches all pending and finalized index data for every secret (from both DB and staged writes). */ + #getSecretsWithPendingData( + jobId: string, + ): Promise<{ secret: string; pendingData: PendingIndexesEntry[]; lastFinalized: number | undefined }[]> { return this.#store.transactionAsync(async () => { // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive const secretDataPromises: Map< string, - { pending: Promise<{ index: number; txHash: string }[]>; finalized: Promise } + { pending: Promise; finalized: Promise } > = new Map(); for await (const secret of this.#pendingIndexes.keysAsync()) { @@ -375,55 +352,125 @@ export class SenderTaggingStore implements StagedStore { })), ); - // Process all txHashes for each secret in memory - for (const { secret, pendingData, lastFinalized } of dataResults) { - if (!pendingData || pendingData.length === 0) { + return dataResults.filter(r => r.pendingData.length > 0); + }); + } + + /** + * Updates pending indexes corresponding to the given transaction hashes to be finalized and prunes any lower pending + * indexes. + */ + async finalizePendingIndexes(txHashes: TxHash[], jobId: string): Promise { + if (txHashes.length === 0) { + return; + } + + const txHashStrings = new Set(txHashes.map(tx => tx.toString())); + const secretsWithData = await this.#getSecretsWithPendingData(jobId); + + for (const { secret, pendingData, lastFinalized } of secretsWithData) { + let currentPending = pendingData; + let currentFinalized = lastFinalized; + + // Process all txHashes for this secret + for (const txHashStr of txHashStrings) { + const matchingEntries = currentPending.filter(item => item.txHash === txHashStr); + if (matchingEntries.length === 0) { + // This is expected as a higher index might have already been finalized which would lead to pruning of + // pending entries. continue; } - let currentPending = pendingData; - let currentFinalized = lastFinalized; + if (matchingEntries.length > 1) { + // We should always just store the highest pending index for a given tx hash and secret because the lower + // values are irrelevant. + throw new Error(`Multiple pending entries found for tx hash ${txHashStr} and secret ${secret}`); + } - // Process all txHashes for this secret - for (const txHashStr of txHashStrings) { - const matchingIndexes = currentPending.filter(item => item.txHash === txHashStr).map(item => item.index); - if (matchingIndexes.length === 0) { - continue; - } + const newFinalized = matchingEntries[0].highestIndex; - if (matchingIndexes.length > 1) { - // We should always just store the highest pending index for a given tx hash and secret because the lower - // values are irrelevant. - throw new Error(`Multiple pending indexes found for tx hash ${txHashStr} and secret ${secret}`); - } + if (newFinalized < (currentFinalized ?? 0)) { + // This should never happen because when last finalized index was finalized we should have pruned the lower + // pending indexes. + throw new Error( + `New finalized index ${newFinalized} is smaller than the current last finalized index ${currentFinalized}`, + ); + } - const newFinalized = matchingIndexes[0]; + currentFinalized = newFinalized; - if (newFinalized < (currentFinalized ?? 0)) { - // This should never happen because when last finalized index was finalized we should have pruned the lower - // pending indexes. - throw new Error( - `New finalized index ${newFinalized} is smaller than the current last finalized index ${currentFinalized}`, - ); - } + // When we add pending indexes, we ensure they are higher than the last finalized index. However, because we + // cannot control the order in which transactions are finalized, there may be pending indexes that are now + // obsolete because they are lower than the most recently finalized index. For this reason, we prune these + // outdated pending indexes. + currentPending = currentPending.filter(item => item.highestIndex > currentFinalized!); + } - currentFinalized = newFinalized; + // Write final state if changed + if (currentFinalized !== lastFinalized) { + this.#writeLastFinalizedIndex(jobId, secret, currentFinalized!); + } + if (currentPending !== pendingData) { + this.#writePendingIndexes(jobId, secret, currentPending); + } + } + } - // When we add pending indexes, we ensure they are higher than the last finalized index. However, because we - // cannot control the order in which transactions are finalized, there may be pending indexes that are now - // obsolete because they are lower than the most recently finalized index. For this reason, we prune these - // outdated pending indexes. - currentPending = currentPending.filter(item => item.index > currentFinalized!); - } + /** + * Handles finalization of pending indexes for a transaction whose execution was partially reverted. + * Recomputes the siloed tags for each pending index of the given tx and checks which ones appear in the + * TxEffect's private logs (i.e., which ones made it onchain). Those that survived are finalized; those that + * didn't are dropped. + * @param txEffect - The tx effect of the partially reverted transaction. + * @param jobId - job context for staged writes to this store. See `JobCoordinator` for more details. + */ + async finalizePendingIndexesOfAPartiallyRevertedTx(txEffect: TxEffect, jobId: string): Promise { + const txHashStr = txEffect.txHash.toString(); - // Write final state if changed - if (currentFinalized !== lastFinalized) { - this.#writeLastFinalizedIndex(jobId, secret, currentFinalized!); - } - if (currentPending !== pendingData) { - this.#writePendingIndexes(jobId, secret, currentPending); + // Build a set of all siloed tag values that made it onchain (first field of each private log). + const onChainTags = new Set(txEffect.privateLogs.map(log => log.fields[0].toString())); + + const secretsWithData = await this.#getSecretsWithPendingData(jobId); + + for (const { secret, pendingData, lastFinalized } of secretsWithData) { + const matchingEntries = pendingData.filter(item => item.txHash === txHashStr); + if (matchingEntries.length === 0) { + // This is expected as a higher index might have already been finalized which would lead to pruning of + // pending entries. + continue; + } + + if (matchingEntries.length > 1) { + // We should always just store the highest pending index for a given tx hash and secret because the lower + // values are irrelevant. + throw new Error(`Multiple pending entries found for tx hash ${txHashStr} and secret ${secret}`); + } + + const pendingEntry = matchingEntries[0]; + + // Expand each matching entry's range and recompute siloed tags for each index. + const extendedSecret = ExtendedDirectionalAppTaggingSecret.fromString(secret); + let highestSurvivingIndex: number | undefined; + + for (let index = pendingEntry.lowestIndex; index <= pendingEntry.highestIndex; index++) { + const siloedTag = await SiloedTag.compute({ extendedSecret, index }); + if (onChainTags.has(siloedTag.value.toString())) { + highestSurvivingIndex = highestSurvivingIndex !== undefined ? Math.max(highestSurvivingIndex, index) : index; } } - }); + + // Remove all entries for this txHash from pending (both surviving and non-surviving). + let currentPending = pendingData.filter(item => item.txHash !== txHashStr); + + if (highestSurvivingIndex !== undefined) { + const newFinalized = Math.max(lastFinalized ?? 0, highestSurvivingIndex); + this.#writeLastFinalizedIndex(jobId, secret, newFinalized); + + // Prune pending indexes that are now <= the finalized index. + currentPending = currentPending.filter(item => item.highestIndex > newFinalized); + } + + this.#writePendingIndexes(jobId, secret, currentPending); + } } } diff --git a/yarn-project/pxe/src/tagging/index.ts b/yarn-project/pxe/src/tagging/index.ts index ea8c6f80f613..6b812a8f0a47 100644 --- a/yarn-project/pxe/src/tagging/index.ts +++ b/yarn-project/pxe/src/tagging/index.ts @@ -16,4 +16,4 @@ export { getAllPrivateLogsByTags, getAllPublicLogsByTagsFromContract } from './g // Re-export tagging-related types from stdlib export { ExtendedDirectionalAppTaggingSecret, Tag, SiloedTag } from '@aztec/stdlib/logs'; -export { type PreTag } from '@aztec/stdlib/logs'; +export { type PreTag, type TaggingIndexRange } from '@aztec/stdlib/logs'; diff --git a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts index d214b6e50120..dedfacbf5dda 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts @@ -1,10 +1,12 @@ import { BlockNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; +import { RevertCode } from '@aztec/stdlib/avm'; import { BlockHash } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; +import { PrivateLog } from '@aztec/stdlib/logs'; import { randomExtendedDirectionalAppTaggingSecret, randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; -import { TxExecutionResult, TxHash, TxReceipt, TxStatus } from '@aztec/stdlib/tx'; +import { type IndexedTxEffect, TxEffect, TxExecutionResult, TxHash, TxReceipt, TxStatus } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -275,4 +277,68 @@ describe('syncSenderTaggingIndexes', () => { expect(await taggingStore.getLastFinalizedIndex(secret, 'test')).toBe(pendingAndFinalizedIndex); expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(pendingAndFinalizedIndex); }); + + it('handles a partially reverted transaction', async () => { + await setUp(); + + const revertedTxHash = TxHash.random(); + + // Create logs at indexes 4 and 6 for the same (reverted) tx + const tag4 = await computeSiloedTagForIndex(4); + const tag6 = await computeSiloedTagForIndex(6); + + aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + return Promise.resolve( + tags.map((tag: SiloedTag) => { + if (tag.equals(tag4)) { + return [makeLog(revertedTxHash, tag4.value)]; + } else if (tag.equals(tag6)) { + return [makeLog(revertedTxHash, tag6.value)]; + } + return []; + }), + ); + }); + + // Mock getTxReceipt to return FINALIZED with APP_LOGIC_REVERTED + aztecNode.getTxReceipt.mockResolvedValue( + new TxReceipt( + revertedTxHash, + TxStatus.FINALIZED, + TxExecutionResult.APP_LOGIC_REVERTED, + undefined, + undefined, + undefined, + BlockNumber(14), + ), + ); + + // Mock getTxEffect to return a TxEffect where only the tag at index 4 survived (non-revertible phase) + const txEffect = new TxEffect( + RevertCode.APP_LOGIC_REVERTED, + revertedTxHash, + Fr.ZERO, + [Fr.random()], // noteHashes + [Fr.random()], // nullifiers + [], // l2ToL1Msgs + [], // publicDataWrites + [PrivateLog.random(tag4.value)], // only the tag at index 4 survived + [], // publicLogs + [], // contractClassLogs + ); + + aztecNode.getTxEffect.mockResolvedValue({ + data: txEffect, + l2BlockNumber: BlockNumber(14), + l2BlockHash: MOCK_ANCHOR_BLOCK_HASH, + txIndexInBlock: 0, + } as IndexedTxEffect); + + await syncSenderTaggingIndexes(secret, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); + + // Index 4 should be finalized (it survived the partial revert) + expect(await taggingStore.getLastFinalizedIndex(secret, 'test')).toBe(4); + // No pending indexes should remain for this secret + expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(4); + }); }); diff --git a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.ts b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.ts index 87d56d6a46e7..516dc00483ef 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.ts @@ -62,11 +62,29 @@ export async function syncSenderTaggingIndexes( break; } - const { txHashesToFinalize, txHashesToDrop } = await getStatusChangeOfPending(pendingTxHashes, aztecNode); + const { txHashesToFinalize, txHashesToDrop, txHashesWithExecutionReverted } = await getStatusChangeOfPending( + pendingTxHashes, + aztecNode, + ); await taggingStore.dropPendingIndexes(txHashesToDrop, jobId); await taggingStore.finalizePendingIndexes(txHashesToFinalize, jobId); + if (txHashesWithExecutionReverted.length > 0) { + const indexedTxEffects = await Promise.all( + txHashesWithExecutionReverted.map(txHash => aztecNode.getTxEffect(txHash)), + ); + for (const indexedTxEffect of indexedTxEffects) { + if (indexedTxEffect === undefined) { + throw new Error( + 'TxEffect not found for execution-reverted tx. This is either a bug or a reorg has occurred.', + ); + } + + await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(indexedTxEffect.data, jobId); + } + } + // We check if the finalized index has been updated. newFinalizedIndex = await taggingStore.getLastFinalizedIndex(secret, jobId); if (previousFinalizedIndex !== newFinalizedIndex) { diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts index 7fd0fc92e3f3..676b491d8910 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts @@ -51,11 +51,41 @@ describe('getStatusChangeOfPending', () => { ), ); } else if (hash.equals(appLogicRevertedTxHash)) { - return Promise.resolve(new TxReceipt(hash, TxStatus.PROPOSED, TxExecutionResult.APP_LOGIC_REVERTED, undefined)); + return Promise.resolve( + new TxReceipt( + hash, + TxStatus.FINALIZED, + TxExecutionResult.APP_LOGIC_REVERTED, + undefined, + undefined, + undefined, + BlockNumber(10), + ), + ); } else if (hash.equals(teardownRevertedTxHash)) { - return Promise.resolve(new TxReceipt(hash, TxStatus.PROPOSED, TxExecutionResult.TEARDOWN_REVERTED, undefined)); + return Promise.resolve( + new TxReceipt( + hash, + TxStatus.FINALIZED, + TxExecutionResult.TEARDOWN_REVERTED, + undefined, + undefined, + undefined, + BlockNumber(10), + ), + ); } else if (hash.equals(bothRevertedTxHash)) { - return Promise.resolve(new TxReceipt(hash, TxStatus.PROPOSED, TxExecutionResult.BOTH_REVERTED, undefined)); + return Promise.resolve( + new TxReceipt( + hash, + TxStatus.FINALIZED, + TxExecutionResult.BOTH_REVERTED, + undefined, + undefined, + undefined, + BlockNumber(10), + ), + ); } else { throw new Error(`Unexpected tx hash: ${hash.toString()}`); } @@ -74,8 +104,8 @@ describe('getStatusChangeOfPending', () => { ); expect(result.txHashesToFinalize).toEqual([finalizedTxHash]); - expect(result.txHashesToDrop).toEqual([ - droppedTxHash, + expect(result.txHashesToDrop).toEqual([droppedTxHash]); + expect(result.txHashesWithExecutionReverted).toEqual([ appLogicRevertedTxHash, teardownRevertedTxHash, bothRevertedTxHash, @@ -101,6 +131,7 @@ describe('getStatusChangeOfPending', () => { expect(result.txHashesToFinalize).toEqual([txHash]); expect(result.txHashesToDrop).toEqual([]); + expect(result.txHashesWithExecutionReverted).toEqual([]); }); it('does not finalize tx that is only proven', async () => { @@ -123,5 +154,6 @@ describe('getStatusChangeOfPending', () => { // Not finalized yet, so stays pending expect(result.txHashesToFinalize).toEqual([]); expect(result.txHashesToDrop).toEqual([]); + expect(result.txHashesWithExecutionReverted).toEqual([]); }); }); diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts index 8400b16237f3..1fc434d10c35 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts @@ -2,35 +2,50 @@ import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { TxHash, TxStatus } from '@aztec/stdlib/tx'; /** - * Based on receipts obtained from `aztecNode` returns which pending transactions changed their status to finalized or - * dropped. + * Based on receipts obtained from `aztecNode` returns which pending transactions changed their status to finalized, + * dropped, or execution-reverted (but mined). */ export async function getStatusChangeOfPending( pending: TxHash[], aztecNode: AztecNode, -): Promise<{ txHashesToFinalize: TxHash[]; txHashesToDrop: TxHash[] }> { +): Promise<{ + txHashesToFinalize: TxHash[]; + txHashesToDrop: TxHash[]; + txHashesWithExecutionReverted: TxHash[]; +}> { // Get receipts for all pending tx hashes. const receipts = await Promise.all(pending.map(pendingTxHash => aztecNode.getTxReceipt(pendingTxHash))); const txHashesToFinalize: TxHash[] = []; const txHashesToDrop: TxHash[] = []; + const txHashesWithExecutionReverted: TxHash[] = []; for (let i = 0; i < receipts.length; i++) { const receipt = receipts[i]; const txHash = pending[i]; - if (receipt.status === TxStatus.FINALIZED && receipt.hasExecutionSucceeded()) { - // Tx has been included in a block and the corresponding block is finalized --> we mark the indexes as - // finalized. - txHashesToFinalize.push(txHash); - } else if (receipt.isDropped() || receipt.hasExecutionReverted()) { - // Tx was dropped or reverted --> we drop the corresponding pending indexes. - // TODO(#17615): Don't drop pending indexes corresponding to non-revertible phases. + if (receipt.status === TxStatus.FINALIZED) { + // Tx has been included in a block and the corresponding block is finalized + if (receipt.hasExecutionSucceeded()) { + // No part of execution reverted - we just finalize all the indexes. + txHashesToFinalize.push(txHash); + } else if (receipt.hasExecutionReverted()) { + // Tx was mined but execution reverted (app logic, teardown, or both). Some logs from the non-revertible + // phase may still be onchain. We check which tags made it onchain and finalize those; drop the rest. + txHashesWithExecutionReverted.push(txHash); + } else { + // Defensive check - this branch should never be triggered + throw new Error( + 'Both hasExecutionSucceeded and hasExecutionReverted on the receipt returned false. This should never happen and it implies a bug. Please open an issue.', + ); + } + } else if (receipt.isDropped()) { + // Tx was dropped from the mempool --> we drop the corresponding pending indexes. txHashesToDrop.push(txHash); } else { // Tx is still pending, not yet finalized, or was mined successfully but not yet finalized --> we don't do anything. } } - return { txHashesToFinalize, txHashesToDrop }; + return { txHashesToFinalize, txHashesToDrop, txHashesWithExecutionReverted }; } diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts index 789c67c79f8f..572ef56fb88e 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts @@ -1,5 +1,4 @@ import type { Fr } from '@aztec/foundation/curves/bn254'; -import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { BlockHash } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { type ExtendedDirectionalAppTaggingSecret, SiloedTag } from '@aztec/stdlib/logs'; @@ -8,17 +7,15 @@ import { TxHash } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; -import { SenderTaggingStore } from '../../../storage/tagging_store/sender_tagging_store.js'; +import type { SenderTaggingStore } from '../../../storage/tagging_store/sender_tagging_store.js'; import { loadAndStoreNewTaggingIndexes } from './load_and_store_new_tagging_indexes.js'; const MOCK_ANCHOR_BLOCK_HASH = BlockHash.random(); describe('loadAndStoreNewTaggingIndexes', () => { - // Secret to be used on the input of the loadAndStoreNewTaggingIndexes function. let secret: ExtendedDirectionalAppTaggingSecret; - let aztecNode: MockProxy; - let taggingStore: SenderTaggingStore; + let taggingStore: MockProxy; function computeSiloedTagForIndex(index: number) { return SiloedTag.compute({ extendedSecret: secret, index }); @@ -30,30 +27,21 @@ describe('loadAndStoreNewTaggingIndexes', () => { beforeAll(async () => { secret = await randomExtendedDirectionalAppTaggingSecret(); - aztecNode = mock(); }); - // Unlike for secret, app address and aztecNode we need a fresh instance of the tagging data provider for each test. - beforeEach(async () => { - aztecNode.getPrivateLogsByTags.mockReset(); - taggingStore = new SenderTaggingStore(await openTmpStore('test')); + beforeEach(() => { + aztecNode = mock(); + taggingStore = mock(); }); it('no logs found for the given window', async () => { aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { - // No log found for any tag - return Promise.resolve(tags.map((_tag: SiloedTag) => [])); + return Promise.resolve(tags.map(() => [])); }); await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - // Verify that no pending indexes were stored - expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBeUndefined(); - expect(await taggingStore.getLastFinalizedIndex(secret, 'test')).toBeUndefined(); - - // Verify the entire window has no pending tx hashes - const txHashesInWindow = await taggingStore.getTxHashesOfPendingIndexes(secret, 0, 10, 'test'); - expect(txHashesInWindow).toHaveLength(0); + expect(taggingStore.storePendingIndexes).not.toHaveBeenCalled(); }); it('single log found at a specific index', async () => { @@ -67,16 +55,15 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - // Verify that the pending index was stored for this txHash - const txHashesInRange = await taggingStore.getTxHashesOfPendingIndexes(secret, index, index + 1, 'test'); - expect(txHashesInRange).toHaveLength(1); - expect(txHashesInRange[0].equals(txHash)).toBe(true); - - // Verify the last used index is correct - expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(index); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(1); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( + [{ extendedSecret: secret, lowestIndex: index, highestIndex: index }], + txHash, + 'test', + ); }); - it('for multiple logs with same txHash stores the highest index', async () => { + it('for multiple logs with same txHash stores full index range', async () => { const txHash = TxHash.random(); const index1 = 3; const index2 = 7; @@ -98,17 +85,12 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - // Verify that only the highest index (7) was stored for this txHash and secret - const txHashesAtIndex2 = await taggingStore.getTxHashesOfPendingIndexes(secret, index2, index2 + 1, 'test'); - expect(txHashesAtIndex2).toHaveLength(1); - expect(txHashesAtIndex2[0].equals(txHash)).toBe(true); - - // Verify the lower index is not stored separately - const txHashesAtIndex1 = await taggingStore.getTxHashesOfPendingIndexes(secret, index1, index1 + 1, 'test'); - expect(txHashesAtIndex1).toHaveLength(0); - - // Verify the last used index is the highest - expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(index2); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(1); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( + [{ extendedSecret: secret, lowestIndex: index1, highestIndex: index2 }], + txHash, + 'test', + ); }); it('multiple logs with different txHashes', async () => { @@ -134,17 +116,17 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - // Verify that both txHashes have their respective indexes stored - const txHashesAtIndex1 = await taggingStore.getTxHashesOfPendingIndexes(secret, index1, index1 + 1, 'test'); - expect(txHashesAtIndex1).toHaveLength(1); - expect(txHashesAtIndex1[0].equals(txHash1)).toBe(true); - - const txHashesAtIndex2 = await taggingStore.getTxHashesOfPendingIndexes(secret, index2, index2 + 1, 'test'); - expect(txHashesAtIndex2).toHaveLength(1); - expect(txHashesAtIndex2[0].equals(txHash2)).toBe(true); - - // Verify the last used index is the highest - expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(index2); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(2); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( + [{ extendedSecret: secret, lowestIndex: index1, highestIndex: index1 }], + txHash1, + 'test', + ); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( + [{ extendedSecret: secret, lowestIndex: index2, highestIndex: index2 }], + txHash2, + 'test', + ); }); // Expected to happen if sending logs from multiple PXEs at a similar time. @@ -162,15 +144,17 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - // Verify that both txHashes have the same index stored - const txHashesAtIndex = await taggingStore.getTxHashesOfPendingIndexes(secret, index, index + 1, 'test'); - expect(txHashesAtIndex).toHaveLength(2); - const txHashStrings = txHashesAtIndex.map(h => h.toString()); - expect(txHashStrings).toContain(txHash1.toString()); - expect(txHashStrings).toContain(txHash2.toString()); - - // Verify the last used index is correct - expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(index); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(2); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( + [{ extendedSecret: secret, lowestIndex: index, highestIndex: index }], + txHash1, + 'test', + ); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( + [{ extendedSecret: secret, lowestIndex: index, highestIndex: index }], + txHash2, + 'test', + ); }); it('complex scenario: multiple txHashes with multiple indexes', async () => { @@ -178,10 +162,11 @@ describe('loadAndStoreNewTaggingIndexes', () => { const txHash2 = TxHash.random(); const txHash3 = TxHash.random(); - // txHash1 has logs at index 1 and 8 (should store 8) - // txHash2 has logs at index 3 and 5 (should store 5) - // txHash3 has a log at index 9 (should store 9) + // txHash1 has logs at index 1, 2 and 8 → range [1, 8] + // txHash2 has logs at index 3 and 5 → range [3, 5] + // txHash3 has a log at index 9 → range [9, 9] const tag1 = await computeSiloedTagForIndex(1); + const tag2 = await computeSiloedTagForIndex(2); const tag3 = await computeSiloedTagForIndex(3); const tag5 = await computeSiloedTagForIndex(5); const tag8 = await computeSiloedTagForIndex(8); @@ -192,6 +177,8 @@ describe('loadAndStoreNewTaggingIndexes', () => { tags.map((t: SiloedTag) => { if (t.equals(tag1)) { return [makeLog(txHash1, tag1.value)]; + } else if (t.equals(tag2)) { + return [makeLog(txHash1, tag1.value)]; } else if (t.equals(tag3)) { return [makeLog(txHash2, tag3.value)]; } else if (t.equals(tag5)) { @@ -208,27 +195,22 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, 0, 10, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - // Verify txHash1 has highest index 8 (should not be at index 1) - const txHashesAtIndex1 = await taggingStore.getTxHashesOfPendingIndexes(secret, 1, 2, 'test'); - expect(txHashesAtIndex1).toHaveLength(0); - const txHashesAtIndex8 = await taggingStore.getTxHashesOfPendingIndexes(secret, 8, 9, 'test'); - expect(txHashesAtIndex8).toHaveLength(1); - expect(txHashesAtIndex8[0].equals(txHash1)).toBe(true); - - // Verify txHash2 has highest index 5 (should not be at index 3) - const txHashesAtIndex3 = await taggingStore.getTxHashesOfPendingIndexes(secret, 3, 4, 'test'); - expect(txHashesAtIndex3).toHaveLength(0); - const txHashesAtIndex5 = await taggingStore.getTxHashesOfPendingIndexes(secret, 5, 6, 'test'); - expect(txHashesAtIndex5).toHaveLength(1); - expect(txHashesAtIndex5[0].equals(txHash2)).toBe(true); - - // Verify txHash3 has index 9 - const txHashesAtIndex9 = await taggingStore.getTxHashesOfPendingIndexes(secret, 9, 10, 'test'); - expect(txHashesAtIndex9).toHaveLength(1); - expect(txHashesAtIndex9[0].equals(txHash3)).toBe(true); - - // Verify the last used index is the highest - expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(9); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(3); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( + [{ extendedSecret: secret, lowestIndex: 1, highestIndex: 8 }], + txHash1, + 'test', + ); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( + [{ extendedSecret: secret, lowestIndex: 3, highestIndex: 5 }], + txHash2, + 'test', + ); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( + [{ extendedSecret: secret, lowestIndex: 9, highestIndex: 9 }], + txHash3, + 'test', + ); }); it('start is inclusive and end is exclusive', async () => { @@ -256,16 +238,12 @@ describe('loadAndStoreNewTaggingIndexes', () => { await loadAndStoreNewTaggingIndexes(secret, start, end, aztecNode, taggingStore, MOCK_ANCHOR_BLOCK_HASH, 'test'); - // Verify that the log at start (inclusive) was processed - const txHashesAtStart = await taggingStore.getTxHashesOfPendingIndexes(secret, start, start + 1, 'test'); - expect(txHashesAtStart).toHaveLength(1); - expect(txHashesAtStart[0].equals(txHashAtStart)).toBe(true); - - // Verify that the log at end (exclusive) was NOT processed - const txHashesAtEnd = await taggingStore.getTxHashesOfPendingIndexes(secret, end, end + 1, 'test'); - expect(txHashesAtEnd).toHaveLength(0); - - // Verify the last used index is the start index (since end was not processed) - expect(await taggingStore.getLastUsedIndex(secret, 'test')).toBe(start); + // Only the log at start should be stored; end is exclusive + expect(taggingStore.storePendingIndexes).toHaveBeenCalledTimes(1); + expect(taggingStore.storePendingIndexes).toHaveBeenCalledWith( + [{ extendedSecret: secret, lowestIndex: start, highestIndex: start }], + txHashAtStart, + 'test', + ); }); }); diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts index 5558c1097cba..3979f5007189 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts @@ -16,6 +16,7 @@ import { getAllPrivateLogsByTags } from '../../get_all_logs_by_tags.js'; * @param end - The ending index (exclusive) of the window to process. * @param aztecNode - The Aztec node instance to query for logs. * @param taggingStore - The data provider to store pending indexes. + * @param anchorBlockHash - Hash of a block to use as reference block when querying node. * @param jobId - Job identifier, used to keep writes in-memory until they can be persisted in a data integrity * preserving way. */ @@ -34,12 +35,13 @@ export async function loadAndStoreNewTaggingIndexes( ); const txsForTags = await getTxsContainingTags(siloedTagsForWindow, aztecNode, anchorBlockHash); - const highestIndexMap = getTxHighestIndexMap(txsForTags, start, siloedTagsForWindow.length); + const txIndexesMap = getTxIndexesMap(txsForTags, start, siloedTagsForWindow.length); - // Now we iterate over the map, reconstruct the preTags and tx hash and store them in the db. - for (const [txHashStr, highestIndex] of highestIndexMap.entries()) { + // Now we iterate over the map, construct the tagging index ranges and store them in the db. + for (const [txHashStr, indexes] of txIndexesMap.entries()) { const txHash = TxHash.fromString(txHashStr); - await taggingStore.storePendingIndexes([{ extendedSecret, index: highestIndex }], txHash, jobId); + const ranges = [{ extendedSecret, lowestIndex: Math.min(...indexes), highestIndex: Math.max(...indexes) }]; + await taggingStore.storePendingIndexes(ranges, txHash, jobId); } } @@ -56,20 +58,28 @@ async function getTxsContainingTags( return allLogs.map(logs => logs.map(log => log.txHash)); } -// Returns a map of txHash to the highest index for that txHash. -function getTxHighestIndexMap(txHashesForTags: TxHash[][], start: number, count: number): Map { +// Returns a map of txHash to all indexes for that txHash. +function getTxIndexesMap(txHashesForTags: TxHash[][], start: number, count: number): Map { if (txHashesForTags.length !== count) { throw new Error(`Number of tx hashes arrays does not match number of tags. ${txHashesForTags.length} !== ${count}`); } - const highestIndexMap = new Map(); + const indexesMap = new Map(); + // Iterate over indexes for (let i = 0; i < txHashesForTags.length; i++) { const taggingIndex = start + i; const txHashesForTag = txHashesForTags[i]; + // iterate over tx hashes that used that index (tag) for (const txHash of txHashesForTag) { const key = txHash.toString(); - highestIndexMap.set(key, Math.max(highestIndexMap.get(key) ?? 0, taggingIndex)); + const existing = indexesMap.get(key); + // Add the index to the tx's indexes + if (existing) { + existing.push(taggingIndex); + } else { + indexesMap.set(key, [taggingIndex]); + } } } - return highestIndexMap; + return indexesMap; } diff --git a/yarn-project/stdlib/src/logs/index.ts b/yarn-project/stdlib/src/logs/index.ts index 2e25c40da7c3..540d1fe99698 100644 --- a/yarn-project/stdlib/src/logs/index.ts +++ b/yarn-project/stdlib/src/logs/index.ts @@ -1,5 +1,6 @@ export * from './extended_directional_app_tagging_secret.js'; export * from './pre_tag.js'; +export * from './tagging_index_range.js'; export * from './contract_class_log.js'; export * from './public_log.js'; export * from './private_log.js'; diff --git a/yarn-project/stdlib/src/logs/tagging_index_range.ts b/yarn-project/stdlib/src/logs/tagging_index_range.ts new file mode 100644 index 000000000000..6392ac8fd26a --- /dev/null +++ b/yarn-project/stdlib/src/logs/tagging_index_range.ts @@ -0,0 +1,24 @@ +import { schemas } from '@aztec/foundation/schemas'; + +import { z } from 'zod'; + +import { + type ExtendedDirectionalAppTaggingSecret, + ExtendedDirectionalAppTaggingSecretSchema, +} from './extended_directional_app_tagging_secret.js'; + +/** + * Represents a range of tagging indexes for a given extended directional app tagging secret. Used to track the lowest + * and highest indexes used in a transaction for a given (sender, recipient, app/contract) tuple. + */ +export type TaggingIndexRange = { + extendedSecret: ExtendedDirectionalAppTaggingSecret; + lowestIndex: number; + highestIndex: number; +}; + +export const TaggingIndexRangeSchema = z.object({ + extendedSecret: ExtendedDirectionalAppTaggingSecretSchema, + lowestIndex: schemas.Integer, + highestIndex: schemas.Integer, +}); diff --git a/yarn-project/stdlib/src/tx/private_execution_result.ts b/yarn-project/stdlib/src/tx/private_execution_result.ts index 4ddd06352e08..4432901b8901 100644 --- a/yarn-project/stdlib/src/tx/private_execution_result.ts +++ b/yarn-project/stdlib/src/tx/private_execution_result.ts @@ -11,7 +11,7 @@ import { PrivateCircuitPublicInputs } from '../kernel/private_circuit_public_inp import type { IsEmpty } from '../kernel/utils/interfaces.js'; import { sortByCounter } from '../kernel/utils/order_and_comparison.js'; import { ContractClassLog, ContractClassLogFields } from '../logs/contract_class_log.js'; -import { type PreTag, PreTagSchema } from '../logs/pre_tag.js'; +import { type TaggingIndexRange, TaggingIndexRangeSchema } from '../logs/tagging_index_range.js'; import { Note } from '../note/note.js'; import { type ZodFor, mapSchema, schemas } from '../schemas/index.js'; import { HashedValues } from './hashed_values.js'; @@ -137,8 +137,8 @@ export class PrivateCallExecutionResult { public returnValues: Fr[], /** The offchain effects emitted during execution of this function call via the `emit_offchain_effect` oracle. */ public offchainEffects: { data: Fr[] }[], - /** The pre-tags used in this tx to compute tags for private logs */ - public preTags: PreTag[], + /** The tagging index ranges used in this tx to compute tags for private logs */ + public taggingIndexRanges: TaggingIndexRange[], /** The nested executions. */ public nestedExecutionResults: PrivateCallExecutionResult[], /** @@ -161,7 +161,7 @@ export class PrivateCallExecutionResult { noteHashNullifierCounterMap: mapSchema(z.coerce.number(), z.number()), returnValues: z.array(schemas.Fr), offchainEffects: z.array(z.object({ data: z.array(schemas.Fr) })), - preTags: z.array(PreTagSchema), + taggingIndexRanges: z.array(TaggingIndexRangeSchema), nestedExecutionResults: z.array(z.lazy(() => PrivateCallExecutionResult.schema)), contractClassLogs: z.array(CountedContractClassLog.schema), }) @@ -178,7 +178,7 @@ export class PrivateCallExecutionResult { fields.noteHashNullifierCounterMap, fields.returnValues, fields.offchainEffects, - fields.preTags, + fields.taggingIndexRanges, fields.nestedExecutionResults, fields.contractClassLogs, );