From d8f345ad1020ce8893ae2b4ec96ee5abd5cd2a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Venturo?= Date: Wed, 28 May 2025 20:06:05 +0000 Subject: [PATCH 1/6] Initial sketch --- .../aztec/src/messages/discovery/mod.nr | 6 +- .../src/messages/discovery/partial_notes.nr | 26 ++- .../src/messages/discovery/private_notes.nr | 25 +-- .../aztec/src/messages/processing/mod.nr | 67 ++++++- .../processing/note_pending_validation.nr | 14 ++ ...age_discovery.nr => message_processing.nr} | 58 +----- .../aztec-nr/aztec/src/oracle/mod.nr | 2 +- .../note_pending_validation.ts | 51 +++++ .../pxe_oracle_interface.ts | 29 ++- .../capsule_data_provider.test.ts | 180 +++++++++++++++--- .../capsule_data_provider.ts | 63 +++++- .../src/private/acvm/oracle/oracle.ts | 26 +-- .../src/private/acvm/oracle/typed_oracle.ts | 13 +- .../src/private/execution_data_provider.ts | 11 +- .../src/private/utility_execution_oracle.ts | 24 +-- .../stdlib/src/logs/log_with_tx_data.ts | 2 +- yarn-project/stdlib/src/tx/tx_hash.ts | 4 + yarn-project/txe/src/oracle/txe_oracle.ts | 21 +- .../txe/src/txe_service/txe_service.ts | 23 +-- 19 files changed, 427 insertions(+), 218 deletions(-) create mode 100644 noir-projects/aztec-nr/aztec/src/messages/processing/note_pending_validation.nr rename noir-projects/aztec-nr/aztec/src/oracle/{message_discovery.nr => message_processing.nr} (52%) create mode 100644 yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts diff --git a/noir-projects/aztec-nr/aztec/src/messages/discovery/mod.nr b/noir-projects/aztec-nr/aztec/src/messages/discovery/mod.nr index da24601d11d5..6aba9a15c4c6 100644 --- a/noir-projects/aztec-nr/aztec/src/messages/discovery/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/messages/discovery/mod.nr @@ -7,7 +7,7 @@ pub mod private_notes; use crate::messages::{ discovery::{private_logs::process_private_log, private_notes::MAX_NOTE_PACKED_LEN}, - processing::{get_private_logs, pending_tagged_log::PendingTaggedLog}, + processing::{get_private_logs, pending_tagged_log::PendingTaggedLog, validate_enqueued_notes}, }; pub struct NoteHashAndNullifier { @@ -80,4 +80,8 @@ pub unconstrained fn discover_new_messages( contract_address, compute_note_hash_and_nullifier, ); + + // Finally we validate all notes that were found as part of the previous processes, resulting in them being added to + // PXE's database and retrievable via oracles. + validate_enqueued_notes(contract_address); } diff --git a/noir-projects/aztec-nr/aztec/src/messages/discovery/partial_notes.nr b/noir-projects/aztec-nr/aztec/src/messages/discovery/partial_notes.nr index 0298748136f1..c76c1b4baf5a 100644 --- a/noir-projects/aztec-nr/aztec/src/messages/discovery/partial_notes.nr +++ b/noir-projects/aztec-nr/aztec/src/messages/discovery/partial_notes.nr @@ -3,8 +3,9 @@ use crate::{ messages::{ discovery::{ComputeNoteHashAndNullifier, nonce_discovery::attempt_note_nonce_discovery}, encoding::MAX_MESSAGE_CONTENT_LEN, + processing::enqueue_note_for_validation, }, - oracle::message_discovery::{deliver_note, get_public_log_by_tag}, + oracle::message_processing::get_public_log_by_tag, utils::array, }; @@ -148,20 +149,15 @@ pub unconstrained fn fetch_and_process_public_partial_note_completion_logs( ); discovered_notes.for_each(|discovered_note| { - // TODO:(#10728): decide how to handle notes that fail delivery. This could be due to e.g. a - // temporary node connectivity issue - is simply throwing good enough here? - assert( - deliver_note( - contract_address, - pending_partial_note.storage_slot, - discovered_note.nonce, - complete_packed_note, - discovered_note.note_hash, - discovered_note.inner_nullifier, - log.tx_hash, - pending_partial_note.recipient, - ), - "Failed to deliver note", + enqueue_note_for_validation( + contract_address, + pending_partial_note.storage_slot, + discovered_note.nonce, + complete_packed_note, + discovered_note.note_hash, + discovered_note.inner_nullifier, + log.tx_hash, + pending_partial_note.recipient, ); }); diff --git a/noir-projects/aztec-nr/aztec/src/messages/discovery/private_notes.nr b/noir-projects/aztec-nr/aztec/src/messages/discovery/private_notes.nr index 01942ed02932..924910a9b7b7 100644 --- a/noir-projects/aztec-nr/aztec/src/messages/discovery/private_notes.nr +++ b/noir-projects/aztec-nr/aztec/src/messages/discovery/private_notes.nr @@ -2,8 +2,8 @@ use crate::{ messages::{ discovery::{ComputeNoteHashAndNullifier, nonce_discovery::attempt_note_nonce_discovery}, encoding::MAX_MESSAGE_CONTENT_LEN, + processing::enqueue_note_for_validation, }, - oracle, utils::array, }; use protocol_types::{ @@ -73,20 +73,15 @@ pub unconstrained fn attempt_note_discovery( ); discovered_notes.for_each(|discovered_note| { - // TODO:(#10728): handle notes that fail delivery. This could be due to e.g. a temporary node connectivity - // issue, and we should perhaps not have marked the tag index as taken. - assert( - oracle::message_discovery::deliver_note( - contract_address, - storage_slot, - discovered_note.nonce, - packed_note, - discovered_note.note_hash, - discovered_note.inner_nullifier, - tx_hash, - recipient, - ), - "Failed to deliver note", + enqueue_note_for_validation( + contract_address, + storage_slot, + discovered_note.nonce, + packed_note, + discovered_note.note_hash, + discovered_note.inner_nullifier, + tx_hash, + recipient, ); }); } diff --git a/noir-projects/aztec-nr/aztec/src/messages/processing/mod.nr b/noir-projects/aztec-nr/aztec/src/messages/processing/mod.nr index 6edf358e9e0b..c810f6ac1d8a 100644 --- a/noir-projects/aztec-nr/aztec/src/messages/processing/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/messages/processing/mod.nr @@ -1,22 +1,75 @@ pub(crate) mod pending_tagged_log; +pub(crate) mod note_pending_validation; use crate::{capsules::CapsuleArray, oracle}; -use crate::messages::processing::pending_tagged_log::PendingTaggedLog; +use crate::messages::{ + discovery::private_notes::MAX_NOTE_PACKED_LEN, + processing::{ + note_pending_validation::NotePendingValidation, pending_tagged_log::PendingTaggedLog, + }, +}; use protocol_types::{address::AztecAddress, hash::sha256_to_field}; // Base slot for the pending tagged log array to which the fetch_tagged_logs oracle inserts found private logs. global PENDING_TAGGED_LOG_ARRAY_BASE_SLOT: Field = sha256_to_field("AZTEC_NR::PENDING_TAGGED_LOG_ARRAY_BASE_SLOT".as_bytes()); -/// Returns a `CapsuleArray` with all private logs that were found and need to be processed. +global NOTE_PENDING_VALIDATION_ARRAY_BASE_SLOT: Field = sha256_to_field( + "AZTEC_NR::NOTE_PENDING_VALIDATION_ARRAY_BASE_SLOT".as_bytes(), +); + +/// Searches for private logs emitted by `contract_address` that might contain messages for one of the local accounts, +/// and stores them in a `CapsuleArray` which is then returned. pub(crate) unconstrained fn get_private_logs( contract_address: AztecAddress, ) -> CapsuleArray { // We will eventually perform log discovery via tagging here, but for now we simply call the `fetchTaggedLogs` - // oracle. This makes PXE synchronize tags, download logs and store the pending tagged logs in capsule array which - // are then retrieved and processed here. - oracle::message_discovery::fetch_tagged_logs(PENDING_TAGGED_LOG_ARRAY_BASE_SLOT); + // oracle. This makes PXE synchronize tags, download logs and store the pending tagged logs in a capsule array. + oracle::message_processing::fetch_tagged_logs(PENDING_TAGGED_LOG_ARRAY_BASE_SLOT); + + CapsuleArray::at(contract_address, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT) +} + +/// Informs PXE of a note's existence so that it can later be retrieved by the `getNotes` oracle. The note will be +/// scoped to `contract_address`, meaning other contracts will not be able to access it unless authorized. +/// +/// The packed note is what `getNotes` will later return. PXE indexes notes by `storage_slot`, so this value +/// is typically used to filter notes that correspond to different state variables. `note_hash` and `nullifier` are +/// the inner hashes, i.e. the raw hashes returned by `NoteHash::compute_note_hash` and +/// `NoteHash::compute_nullifier`. PXE will verify that the siloed unique note hash was inserted into the tree +/// at `tx_hash`, and will store the nullifier to later check for nullification. +/// +/// `recipient` is the account to which the note was sent to. Other accounts will not be able to access this note (e.g. +/// other accounts will not be able to see one another's token balance notes, even in the same PXE) unless authorized. +/// +/// Returns true if the note was successfully delivered and added to PXE's database. +pub(crate) unconstrained fn enqueue_note_for_validation( + contract_address: AztecAddress, + storage_slot: Field, + nonce: Field, + packed_note: BoundedVec, + note_hash: Field, + nullifier: Field, + tx_hash: Field, + recipient: AztecAddress, +) { + CapsuleArray::at(contract_address, NOTE_PENDING_VALIDATION_ARRAY_BASE_SLOT).push( + NotePendingValidation { + contract_address, + storage_slot, + nonce, + packed_note, + note_hash, + nullifier, + tx_hash, + recipient, + }, + ) +} - // Get the logs from the capsule array and process them one by one - CapsuleArray::::at(contract_address, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT) +pub(crate) unconstrained fn validate_enqueued_notes(contract_address: AztecAddress) { + oracle::message_processing::validate_enqueued_notes( + contract_address, + NOTE_PENDING_VALIDATION_ARRAY_BASE_SLOT, + ); } diff --git a/noir-projects/aztec-nr/aztec/src/messages/processing/note_pending_validation.nr b/noir-projects/aztec-nr/aztec/src/messages/processing/note_pending_validation.nr new file mode 100644 index 000000000000..6c31910b07dd --- /dev/null +++ b/noir-projects/aztec-nr/aztec/src/messages/processing/note_pending_validation.nr @@ -0,0 +1,14 @@ +use crate::messages::discovery::private_notes::MAX_NOTE_PACKED_LEN; +use protocol_types::{address::AztecAddress, traits::Serialize}; + +#[derive(Serialize)] +pub(crate) struct NotePendingValidation { + pub contract_address: AztecAddress, + pub storage_slot: Field, + pub nonce: Field, + pub packed_note: BoundedVec, + pub note_hash: Field, + pub nullifier: Field, + pub tx_hash: Field, + pub recipient: AztecAddress, +} diff --git a/noir-projects/aztec-nr/aztec/src/oracle/message_discovery.nr b/noir-projects/aztec-nr/aztec/src/oracle/message_processing.nr similarity index 52% rename from noir-projects/aztec-nr/aztec/src/oracle/message_discovery.nr rename to noir-projects/aztec-nr/aztec/src/oracle/message_processing.nr index 5a4ae812df2e..849d4a5700e7 100644 --- a/noir-projects/aztec-nr/aztec/src/oracle/message_discovery.nr +++ b/noir-projects/aztec-nr/aztec/src/oracle/message_processing.nr @@ -1,4 +1,3 @@ -use crate::messages::discovery::private_notes::MAX_NOTE_PACKED_LEN; use dep::protocol_types::{ address::AztecAddress, constants::{MAX_NOTE_HASHES_PER_TX, PUBLIC_LOG_SIZE_IN_FIELDS}, @@ -13,41 +12,6 @@ pub unconstrained fn fetch_tagged_logs(pending_tagged_log_array_base_slot: Field #[oracle(fetchTaggedLogs)] unconstrained fn fetch_tagged_logs_oracle(pending_tagged_log_array_base_slot: Field) {} -/// Informs PXE of a note's existence so that it can later be retrieved by the `getNotes` oracle. The note will be -/// scoped to `contract_address`, meaning other contracts will not be able to access it unless authorized. -/// -/// The packed note is what `getNotes` will later return. PXE indexes notes by `storage_slot`, so this value -/// is typically used to filter notes that correspond to different state variables. `note_hash` and `nullifier` are -/// the inner hashes, i.e. the raw hashes returned by `NoteHash::compute_note_hash` and -/// `NoteHash::compute_nullifier`. PXE will verify that the siloed unique note hash was inserted into the tree -/// at `tx_hash`, and will store the nullifier to later check for nullification. -/// -/// `recipient` is the account to which the note was sent to. Other accounts will not be able to access this note (e.g. -/// other accounts will not be able to see one another's token balance notes, even in the same PXE) unless authorized. -/// -/// Returns true if the note was successfully delivered and added to PXE's database. -pub unconstrained fn deliver_note( - contract_address: AztecAddress, - storage_slot: Field, - nonce: Field, - packed_note: BoundedVec, - note_hash: Field, - nullifier: Field, - tx_hash: Field, - recipient: AztecAddress, -) -> bool { - deliver_note_oracle( - contract_address, - storage_slot, - nonce, - packed_note, - note_hash, - nullifier, - tx_hash, - recipient, - ) -} - /// The contents of a public log, plus contextual information about the transaction in which the log was emitted. This /// is the data required in order to discover notes that are being delivered in a log. pub struct PublicLogWithTxData { @@ -71,20 +35,18 @@ pub unconstrained fn get_public_log_by_tag( get_public_log_by_tag_oracle(tag, contract_address) } -#[oracle(deliverNote)] -unconstrained fn deliver_note_oracle( - contract_address: AztecAddress, - storage_slot: Field, - nonce: Field, - packed_note: BoundedVec, - note_hash: Field, - nullifier: Field, - tx_hash: Field, - recipient: AztecAddress, -) -> bool {} - #[oracle(getPublicLogByTag)] unconstrained fn get_public_log_by_tag_oracle( tag: Field, contract_address: AztecAddress, ) -> Option {} + +pub(crate) unconstrained fn validate_enqueued_notes( + contract_address: AztecAddress, + base_slot: Field, +) { + validate_enqueued_notes_oracle(contract_address, base_slot); +} + +#[oracle(validateEnqueuedNotes)] +unconstrained fn validate_enqueued_notes_oracle(contract_address: AztecAddress, base_slot: Field) {} diff --git a/noir-projects/aztec-nr/aztec/src/oracle/mod.nr b/noir-projects/aztec-nr/aztec/src/oracle/mod.nr index 46249dafc0e7..4faa5a80e077 100644 --- a/noir-projects/aztec-nr/aztec/src/oracle/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/oracle/mod.nr @@ -18,7 +18,7 @@ pub mod get_membership_witness; pub mod keys; pub mod key_validation_request; pub mod logs; -pub mod message_discovery; +pub mod message_processing; pub mod notes; pub mod random; pub mod shared_secret; diff --git a/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts b/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts new file mode 100644 index 000000000000..96f2fd2d7d03 --- /dev/null +++ b/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts @@ -0,0 +1,51 @@ +import { Fr } from '@aztec/foundation/fields'; +import { FieldReader } from '@aztec/foundation/serialize'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; +import { TxHash } from '@aztec/stdlib/tx'; + +const MAX_NOTE_PACKED_LEN = 12; // TODO ?? + +/** + * Represents a pending tagged log as it is stored in the pending tagged log array to which the fetchTaggedLogs oracle + * inserts found private logs. A TS version of `pending_tagged_log.nr`. + */ +export class NotePendingValidation { + constructor( + public contractAddress: AztecAddress, + public storageSlot: Fr, + public nonce: Fr, + public content: Fr[], + public noteHash: Fr, + public nullifier: Fr, + public txHash: TxHash, + public recipient: AztecAddress, + ) {} + + static fromFields(fields: Fr[] | FieldReader): NotePendingValidation { + const reader = FieldReader.asReader(fields); + + const contractAddress = AztecAddress.fromField(reader.readField()); + const storageSlot = reader.readField(); + const nonce = reader.readField(); + + const contentLen = reader.readField().toNumber(); + const contentStorage = reader.readFieldArray(MAX_NOTE_PACKED_LEN); + const content = contentStorage.slice(0, contentLen); + + const noteHash = reader.readField(); + const nullifier = reader.readField(); + const txHash = TxHash.fromField(reader.readField()); + const recipient = AztecAddress.fromField(reader.readField()); + + return new NotePendingValidation( + contractAddress, + storageSlot, + nonce, + content, + noteHash, + nullifier, + txHash, + recipient, + ); + } +} diff --git a/yarn-project/pxe/src/pxe_oracle_interface/pxe_oracle_interface.ts b/yarn-project/pxe/src/pxe_oracle_interface/pxe_oracle_interface.ts index 48ff18828437..45f2147d2716 100644 --- a/yarn-project/pxe/src/pxe_oracle_interface/pxe_oracle_interface.ts +++ b/yarn-project/pxe/src/pxe_oracle_interface/pxe_oracle_interface.ts @@ -39,6 +39,7 @@ import type { NoteDataProvider } from '../storage/note_data_provider/note_data_p import type { PrivateEventDataProvider } from '../storage/private_event_data_provider/private_event_data_provider.js'; import type { SyncDataProvider } from '../storage/sync_data_provider/sync_data_provider.js'; import type { TaggingDataProvider } from '../storage/tagging_data_provider/tagging_data_provider.js'; +import { NotePendingValidation } from './message_processing/note_pending_validation.js'; import { WINDOW_HALF_SIZE, getIndexedTaggingSecretsForTheWindow, getInitialIndexesMap } from './tagging_utils.js'; /** @@ -608,7 +609,33 @@ export class PXEOracleInterface implements ExecutionDataProvider { return this.capsuleDataProvider.appendToCapsuleArray(contractAddress, capsuleArrayBaseSlot, pendingTaggedLogs); } - public async deliverNote( + public async validateEnqueuedNotes( + contractAddress: AztecAddress, + notePendingValidationArrayBaseSlot: Fr, + ): Promise { + const notesPendingValidation = ( + await this.capsuleDataProvider.readCapsuleArray(contractAddress, notePendingValidationArrayBaseSlot) + ).map(NotePendingValidation.fromFields); + + await Promise.all( + notesPendingValidation.map(note => + this.deliverNote( + note.contractAddress, + note.storageSlot, + note.nonce, + note.content, + note.noteHash, + note.nullifier, + note.txHash, + note.recipient, + ), + ), + ); + + await this.capsuleDataProvider.resetCapsuleArray(contractAddress, notePendingValidationArrayBaseSlot, []); + } + + async deliverNote( contractAddress: AztecAddress, storageSlot: Fr, nonce: Fr, diff --git a/yarn-project/pxe/src/storage/capsule_data_provider/capsule_data_provider.test.ts b/yarn-project/pxe/src/storage/capsule_data_provider/capsule_data_provider.test.ts index bb616b40ee78..51d0250f5d28 100644 --- a/yarn-project/pxe/src/storage/capsule_data_provider/capsule_data_provider.test.ts +++ b/yarn-project/pxe/src/storage/capsule_data_provider/capsule_data_provider.test.ts @@ -177,36 +177,130 @@ describe('capsule data provider', () => { }); }); - describe('append to capsule array', () => { - it('creates a new array', async () => { - const baseSlot = new Fr(3); - const array = range(4).map(x => [new Fr(x)]); + describe('arrays', () => { + describe('appendToCapsuleArray', () => { + it('creates a new array', async () => { + const baseSlot = new Fr(3); + const array = range(4).map(x => [new Fr(x)]); + + await capsuleDataProvider.appendToCapsuleArray(contract, baseSlot, array); + + expect(await capsuleDataProvider.loadCapsule(contract, baseSlot)).toEqual([new Fr(array.length)]); + for (const i of range(array.length)) { + expect(await capsuleDataProvider.loadCapsule(contract, baseSlot.add(new Fr(1 + i)))).toEqual(array[i]); + } + }); + + it('appends to an existing array', async () => { + const baseSlot = new Fr(3); + const originalArray = range(4).map(x => [new Fr(x)]); + + await capsuleDataProvider.appendToCapsuleArray(contract, baseSlot, originalArray); + + const newElements = [[new Fr(13)], [new Fr(42)]]; + await capsuleDataProvider.appendToCapsuleArray(contract, baseSlot, newElements); + + const expectedLength = originalArray.length + newElements.length; + + expect(await capsuleDataProvider.loadCapsule(contract, baseSlot)).toEqual([new Fr(expectedLength)]); + for (const i of range(expectedLength)) { + expect(await capsuleDataProvider.loadCapsule(contract, baseSlot.add(new Fr(1 + i)))).toEqual( + [...originalArray, ...newElements][i], + ); + } + }); + }); - await capsuleDataProvider.appendToCapsuleArray(contract, baseSlot, array); + describe('readCapsuleArray', () => { + it('reads an empty array', async () => { + const baseSlot = new Fr(3); + const retrievedArray = await capsuleDataProvider.readCapsuleArray(contract, baseSlot); + expect(retrievedArray).toEqual([]); + }); - expect(await capsuleDataProvider.loadCapsule(contract, baseSlot)).toEqual([new Fr(array.length)]); - for (const i of range(array.length)) { - expect(await capsuleDataProvider.loadCapsule(contract, baseSlot.add(new Fr(1 + i)))).toEqual(array[i]); - } - }); + it('reads an existing array', async () => { + const baseSlot = new Fr(3); + const storedArray = range(4).map(x => [new Fr(x)]); - it('appends to an existing array', async () => { - const baseSlot = new Fr(3); - const originalArray = range(4).map(x => [new Fr(x)]); + await capsuleDataProvider.appendToCapsuleArray(contract, baseSlot, storedArray); - await capsuleDataProvider.appendToCapsuleArray(contract, baseSlot, originalArray); + const retrievedArray = await capsuleDataProvider.readCapsuleArray(contract, baseSlot); + expect(retrievedArray).toEqual(storedArray); + }); - const newElements = [[new Fr(13)], [new Fr(42)]]; - await capsuleDataProvider.appendToCapsuleArray(contract, baseSlot, newElements); + it('throws on a corrupted array', async () => { + const baseSlot = new Fr(3); - const expectedLength = originalArray.length + newElements.length; + // Store in the base slot a non-zero value, indicating a non-zero array length + await capsuleDataProvider.storeCapsule(contract, baseSlot, [new Fr(1)]); - expect(await capsuleDataProvider.loadCapsule(contract, baseSlot)).toEqual([new Fr(expectedLength)]); - for (const i of range(expectedLength)) { - expect(await capsuleDataProvider.loadCapsule(contract, baseSlot.add(new Fr(1 + i)))).toEqual( - [...originalArray, ...newElements][i], + // Reading should now fail as some of the capsules in the array are empty + await expect(capsuleDataProvider.readCapsuleArray(contract, baseSlot)).rejects.toThrow( + 'Expected non-empty value', ); - } + }); + }); + + describe('resetCapsuleArray', () => { + it('resets an empty array', async () => { + const baseSlot = new Fr(3); + const newArray = range(4).map(x => [new Fr(x)]); + + await capsuleDataProvider.resetCapsuleArray(contract, baseSlot, newArray); + + const retrievedArray = await capsuleDataProvider.readCapsuleArray(contract, baseSlot); + expect(retrievedArray).toEqual(newArray); + }); + + it('resets an existing shorter array', async () => { + const baseSlot = new Fr(3); + + const originalArray = range(4, 0).map(x => [new Fr(x)]); + await capsuleDataProvider.resetCapsuleArray(contract, baseSlot, originalArray); + + const newArray = range(10, 10).map(x => [new Fr(x)]); + await capsuleDataProvider.resetCapsuleArray(contract, baseSlot, newArray); + + const retrievedArray = await capsuleDataProvider.readCapsuleArray(contract, baseSlot); + expect(retrievedArray).toEqual(newArray); + }); + + it('resets an existing longer array', async () => { + const baseSlot = new Fr(3); + + const originalArray = range(10, 0).map(x => [new Fr(x)]); + await capsuleDataProvider.resetCapsuleArray(contract, baseSlot, originalArray); + + const newArray = range(4, 10).map(x => [new Fr(x)]); + await capsuleDataProvider.resetCapsuleArray(contract, baseSlot, newArray); + + const retrievedArray = await capsuleDataProvider.readCapsuleArray(contract, baseSlot); + expect(retrievedArray).toEqual(newArray); + + // Not only do we read the expected array, but also all capsules past the new array length have been cleared + for (const i of range(originalArray.length - newArray.length)) { + expect( + await capsuleDataProvider.loadCapsule(contract, baseSlot.add(new Fr(1 + newArray.length + i))), + ).toBeNull(); + } + }); + + it('clears an existing array', async () => { + const baseSlot = new Fr(3); + + const originalArray = range(10, 0).map(x => [new Fr(x)]); + await capsuleDataProvider.resetCapsuleArray(contract, baseSlot, originalArray); + + await capsuleDataProvider.resetCapsuleArray(contract, baseSlot, []); + + const retrievedArray = await capsuleDataProvider.readCapsuleArray(contract, baseSlot); + expect(retrievedArray).toEqual([]); + + // All capsules from the original array have been cleared + for (const i of range(originalArray.length)) { + expect(await capsuleDataProvider.loadCapsule(contract, baseSlot.add(new Fr(1 + i)))).toBeNull(); + } + }); }); }); @@ -225,7 +319,7 @@ describe('capsule data provider', () => { const ARRAY_LENGTH = 50; it( - 'create large array', + 'create large array by appending', async () => { await capsuleDataProvider.appendToCapsuleArray( contract, @@ -236,6 +330,18 @@ describe('capsule data provider', () => { TEST_TIMEOUT_MS, ); + it( + 'create large array by resetting', + async () => { + await capsuleDataProvider.resetCapsuleArray( + contract, + new Fr(0), + times(NUMBER_OF_ITEMS, () => range(ARRAY_LENGTH).map(x => new Fr(x))), + ); + }, + TEST_TIMEOUT_MS, + ); + it( 'append to large array', async () => { @@ -265,5 +371,33 @@ describe('capsule data provider', () => { }, TEST_TIMEOUT_MS, ); + + it( + 'read a large array', + async () => { + await capsuleDataProvider.appendToCapsuleArray( + contract, + new Fr(0), + times(NUMBER_OF_ITEMS, () => range(ARRAY_LENGTH).map(x => new Fr(x))), + ); + + await capsuleDataProvider.readCapsuleArray(contract, new Fr(0)); + }, + TEST_TIMEOUT_MS, + ); + + it( + 'clear a large array', + async () => { + await capsuleDataProvider.appendToCapsuleArray( + contract, + new Fr(0), + times(NUMBER_OF_ITEMS, () => range(ARRAY_LENGTH).map(x => new Fr(x))), + ); + + await capsuleDataProvider.resetCapsuleArray(contract, new Fr(0), []); + }, + TEST_TIMEOUT_MS, + ); }); }); diff --git a/yarn-project/pxe/src/storage/capsule_data_provider/capsule_data_provider.ts b/yarn-project/pxe/src/storage/capsule_data_provider/capsule_data_provider.ts index f65edd39bde7..9ce111f8b89a 100644 --- a/yarn-project/pxe/src/storage/capsule_data_provider/capsule_data_provider.ts +++ b/yarn-project/pxe/src/storage/capsule_data_provider/capsule_data_provider.ts @@ -75,26 +75,71 @@ export class CapsuleDataProvider implements DataProvider { * All operations are performed in a single transaction. * @param contractAddress - The contract address that owns the capsule array * @param baseSlot - The slot where the array length is stored - * @param capsules - Array of capsule data to append + * @param content - Array of capsule data to append */ - appendToCapsuleArray(contractAddress: AztecAddress, baseSlot: Fr, capsules: Fr[][]): Promise { + appendToCapsuleArray(contractAddress: AztecAddress, baseSlot: Fr, content: Fr[][]): Promise { return this.#store.transactionAsync(async () => { // Load current length, defaulting to 0 if not found const lengthData = await this.loadCapsule(contractAddress, baseSlot); - const currentLength = lengthData ? lengthData[0].toBigInt() : 0n; + const currentLength = lengthData ? lengthData[0].toNumber() : 0; // Store each capsule at consecutive slots after baseSlot + 1 + currentLength - for (let i = 0; i < capsules.length; i++) { - const nextSlot = baseSlot.add(new Fr(1)).add(new Fr(currentLength + BigInt(i))); - await this.storeCapsule(contractAddress, nextSlot, capsules[i]); + for (let i = 0; i < content.length; i++) { + const nextSlot = arraySlot(baseSlot, currentLength + i); + await this.storeCapsule(contractAddress, nextSlot, content[i]); } // Update length to include all new capsules - const newLength = currentLength + BigInt(capsules.length); + const newLength = currentLength + content.length; await this.storeCapsule(contractAddress, baseSlot, [new Fr(newLength)]); }); } + readCapsuleArray(contractAddress: AztecAddress, baseSlot: Fr): Promise { + return this.#store.transactionAsync(async () => { + // Load length, defaulting to 0 if not found + const maybeLength = await this.loadCapsule(contractAddress, baseSlot); + const length = maybeLength ? maybeLength[0].toBigInt() : 0n; + + const values: Fr[][] = []; + + // Read each capsule at consecutive slots after baseSlot + for (let i = 0; i < length; i++) { + const currentValue = await this.loadCapsule(contractAddress, arraySlot(baseSlot, i)); + if (currentValue == undefined) { + throw new Error( + `Expected non-empty value at capsule array in base slot ${baseSlot} at index ${i} for contract ${contractAddress}`, + ); + } + + values.push(currentValue); + } + + return values; + }); + } + + resetCapsuleArray(contractAddress: AztecAddress, baseSlot: Fr, content: Fr[][]) { + return this.#store.transactionAsync(async () => { + // Load current length, defaulting to 0 if not found + const maybeLength = await this.loadCapsule(contractAddress, baseSlot); + const originalLength = maybeLength ? maybeLength[0].toNumber() : 0; + + // Set the new length + await this.storeCapsule(contractAddress, baseSlot, [new Fr(content.length)]); + + // Store the new content, possibly overwriting existing values + for (let i = 0; i < content.length; i++) { + await this.storeCapsule(contractAddress, arraySlot(baseSlot, i), content[i]); + } + + // Clear any stragglers + for (let i = content.length; i < originalLength; i++) { + await this.deleteCapsule(contractAddress, arraySlot(baseSlot, i)); + } + }); + } + public async getSize() { return (await toArray(this.#capsules.valuesAsync())).reduce( (sum, value) => sum + value.length * Fr.SIZE_IN_BYTES, @@ -106,3 +151,7 @@ export class CapsuleDataProvider implements DataProvider { function dbSlotToKey(contractAddress: AztecAddress, slot: Fr): string { return `${contractAddress.toString()}:${slot.toString()}`; } + +function arraySlot(baseSlot: Fr, index: number) { + return baseSlot.add(new Fr(1)).add(new Fr(index)); +} diff --git a/yarn-project/simulator/src/private/acvm/oracle/oracle.ts b/yarn-project/simulator/src/private/acvm/oracle/oracle.ts index 35b5da6aca74..a66635805852 100644 --- a/yarn-project/simulator/src/private/acvm/oracle/oracle.ts +++ b/yarn-project/simulator/src/private/acvm/oracle/oracle.ts @@ -384,32 +384,16 @@ export class Oracle { return []; } - async deliverNote( + async validateEnqueuedNotes( [contractAddress]: ACVMField[], - [storageSlot]: ACVMField[], - [nonce]: ACVMField[], - content: ACVMField[], - [contentLength]: ACVMField[], - [noteHash]: ACVMField[], - [nullifier]: ACVMField[], - [txHash]: ACVMField[], - [recipient]: ACVMField[], + [notePendingValidationArrayBaseSlot]: ACVMField[], ): Promise { - // TODO(#10728): try-catch this block and return false if we get an exception so that the contract can decide what - // to do if a note fails delivery (e.g. not increment the tagging index, or add it to some pending work list). - // Delivery might fail due to temporary issues, such as poor node connectivity. - await this.typedOracle.deliverNote( + await this.typedOracle.validateEnqueuedNotes( AztecAddress.fromString(contractAddress), - Fr.fromString(storageSlot), - Fr.fromString(nonce), - fromBoundedVec(content, contentLength), - Fr.fromString(noteHash), - Fr.fromString(nullifier), - TxHash.fromString(txHash), - AztecAddress.fromString(recipient), + Fr.fromString(notePendingValidationArrayBaseSlot), ); - return [toACVMField(true)]; + return []; } async getPublicLogByTag([tag]: ACVMField[], [contractAddress]: ACVMField[]): Promise<(ACVMField | ACVMField[])[]> { diff --git a/yarn-project/simulator/src/private/acvm/oracle/typed_oracle.ts b/yarn-project/simulator/src/private/acvm/oracle/typed_oracle.ts index f1a6d77e58e0..7f273d932b4f 100644 --- a/yarn-project/simulator/src/private/acvm/oracle/typed_oracle.ts +++ b/yarn-project/simulator/src/private/acvm/oracle/typed_oracle.ts @@ -218,17 +218,8 @@ export abstract class TypedOracle { return Promise.reject(new OracleMethodNotAvailableError('fetchTaggedLogs')); } - deliverNote( - _contractAddress: AztecAddress, - _storageSlot: Fr, - _nonce: Fr, - _content: Fr[], - _noteHash: Fr, - _nullifier: Fr, - _txHash: TxHash, - _recipient: AztecAddress, - ): Promise { - return Promise.reject(new OracleMethodNotAvailableError('deliverNote')); + validateEnqueuedNotes(_contractAddress: AztecAddress, _notePendingValidationArrayBaseSlot: Fr): Promise { + return Promise.reject(new OracleMethodNotAvailableError('validateEnqueuedNotes')); } getPublicLogByTag(_tag: Fr, _contractAddress: AztecAddress): Promise { diff --git a/yarn-project/simulator/src/private/execution_data_provider.ts b/yarn-project/simulator/src/private/execution_data_provider.ts index ce1fd99ec328..ab8213512c20 100644 --- a/yarn-project/simulator/src/private/execution_data_provider.ts +++ b/yarn-project/simulator/src/private/execution_data_provider.ts @@ -287,16 +287,7 @@ export interface ExecutionDataProvider { * @param txHash - The transaction in which the note was added to the note hash tree * @param recipient - The account that discovered the note */ - deliverNote( - contractAddress: AztecAddress, - storageSlot: Fr, - nonce: Fr, - content: Fr[], - noteHash: Fr, - nullifier: Fr, - txHash: TxHash, - recipient: AztecAddress, - ): Promise; + validateEnqueuedNotes(contractAddress: AztecAddress, notePendingValidationArrayBaseSlot: Fr): Promise; /** * Gets note hash in the note hash tree at the given leaf index. diff --git a/yarn-project/simulator/src/private/utility_execution_oracle.ts b/yarn-project/simulator/src/private/utility_execution_oracle.ts index ce10987e5f80..b37a7b7dd003 100644 --- a/yarn-project/simulator/src/private/utility_execution_oracle.ts +++ b/yarn-project/simulator/src/private/utility_execution_oracle.ts @@ -280,31 +280,13 @@ export class UtilityExecutionOracle extends TypedOracle { await this.executionDataProvider.removeNullifiedNotes(this.contractAddress); } - public override async deliverNote( - contractAddress: AztecAddress, - storageSlot: Fr, - nonce: Fr, - content: Fr[], - noteHash: Fr, - nullifier: Fr, - txHash: TxHash, - recipient: AztecAddress, - ) { + public override async validateEnqueuedNotes(contractAddress: AztecAddress, notePendingValidationArrayBaseSlot: Fr) { // TODO(#10727): allow other contracts to deliver notes if (!this.contractAddress.equals(contractAddress)) { - throw new Error(`Got a note delivery request from ${contractAddress}, expected ${this.contractAddress}`); + throw new Error(`Got a note validation request from ${contractAddress}, expected ${this.contractAddress}`); } - await this.executionDataProvider.deliverNote( - contractAddress, - storageSlot, - nonce, - content, - noteHash, - nullifier, - txHash, - recipient, - ); + await this.executionDataProvider.validateEnqueuedNotes(contractAddress, notePendingValidationArrayBaseSlot); } public override getPublicLogByTag(tag: Fr, contractAddress: AztecAddress): Promise { diff --git a/yarn-project/stdlib/src/logs/log_with_tx_data.ts b/yarn-project/stdlib/src/logs/log_with_tx_data.ts index b5fa2e15f948..cf3b74061e5d 100644 --- a/yarn-project/stdlib/src/logs/log_with_tx_data.ts +++ b/yarn-project/stdlib/src/logs/log_with_tx_data.ts @@ -2,7 +2,7 @@ import { MAX_NOTE_HASHES_PER_TX, PUBLIC_LOG_SIZE_IN_FIELDS } from '@aztec/consta import { Fr } from '@aztec/foundation/fields'; import { TxHash } from '@aztec/stdlib/tx'; -// TypeScript representation of the Noir aztec::oracle::message_discovery::PublicLogWithTxData struct. This is used as a +// TypeScript representation of the Noir aztec::oracle::message_processing::PublicLogWithTxData struct. This is used as a // response for PXE's custom getPublicLogByTag oracle. export class PublicLogWithTxData { constructor( diff --git a/yarn-project/stdlib/src/tx/tx_hash.ts b/yarn-project/stdlib/src/tx/tx_hash.ts index 01d52a072e6b..dcbcf132eceb 100644 --- a/yarn-project/stdlib/src/tx/tx_hash.ts +++ b/yarn-project/stdlib/src/tx/tx_hash.ts @@ -29,6 +29,10 @@ export class TxHash { return new TxHash(new Fr(value)); } + static fromField(value: Fr) { + return new TxHash(value); + } + public toBuffer() { return this.hash.toBuffer(); } diff --git a/yarn-project/txe/src/oracle/txe_oracle.ts b/yarn-project/txe/src/oracle/txe_oracle.ts index c13a48b223f8..60960ccb7fe5 100644 --- a/yarn-project/txe/src/oracle/txe_oracle.ts +++ b/yarn-project/txe/src/oracle/txe_oracle.ts @@ -1176,26 +1176,11 @@ export class TXE implements TypedOracle { return Promise.resolve(); } - public async deliverNote( + public async validateEnqueuedNotes( contractAddress: AztecAddress, - storageSlot: Fr, - nonce: Fr, - content: Fr[], - noteHash: Fr, - nullifier: Fr, - txHash: TxHash, - recipient: AztecAddress, + notePendingValidationArrayBaseSlot: Fr, ): Promise { - await this.pxeOracleInterface.deliverNote( - contractAddress, - storageSlot, - nonce, - content, - noteHash, - nullifier, - txHash, - recipient, - ); + await this.pxeOracleInterface.validateEnqueuedNotes(contractAddress, notePendingValidationArrayBaseSlot); } async getPublicLogByTag(tag: Fr, contractAddress: AztecAddress): Promise { diff --git a/yarn-project/txe/src/txe_service/txe_service.ts b/yarn-project/txe/src/txe_service/txe_service.ts index babce60cd599..93fe8fe256a2 100644 --- a/yarn-project/txe/src/txe_service/txe_service.ts +++ b/yarn-project/txe/src/txe_service/txe_service.ts @@ -700,16 +700,9 @@ export class TXEService { return toForeignCallResult([]); } - public async deliverNote( + public async validateEnqueuedNotes( contractAddress: ForeignCallSingle, - storageSlot: ForeignCallSingle, - nonce: ForeignCallSingle, - content: ForeignCallArray, - contentLength: ForeignCallSingle, - noteHash: ForeignCallSingle, - nullifier: ForeignCallSingle, - txHash: ForeignCallSingle, - recipient: ForeignCallSingle, + notePendingValidationArrayBaseSlot: ForeignCallSingle, ) { if (!this.oraclesEnabled) { throw new Error( @@ -717,18 +710,12 @@ export class TXEService { ); } - await this.typedOracle.deliverNote( + await this.typedOracle.validateEnqueuedNotes( AztecAddress.fromField(fromSingle(contractAddress)), - fromSingle(storageSlot), - fromSingle(nonce), - fromArray(content.slice(0, Number(BigInt(contentLength)))), - fromSingle(noteHash), - fromSingle(nullifier), - new TxHash(fromSingle(txHash)), - AztecAddress.fromField(fromSingle(recipient)), + fromSingle(notePendingValidationArrayBaseSlot), ); - return toForeignCallResult([toSingle(Fr.ONE)]); + return toForeignCallResult([]); } async getPublicLogByTag(tag: ForeignCallSingle, contractAddress: ForeignCallSingle) { From fea2d36842dd5cb7b466d45944848f854cc3db13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Venturo?= Date: Wed, 28 May 2025 20:31:33 +0000 Subject: [PATCH 2/6] Fix deserialization --- .../message_processing/note_pending_validation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts b/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts index 96f2fd2d7d03..c6c0d41ac28b 100644 --- a/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts +++ b/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts @@ -28,8 +28,8 @@ export class NotePendingValidation { const storageSlot = reader.readField(); const nonce = reader.readField(); - const contentLen = reader.readField().toNumber(); const contentStorage = reader.readFieldArray(MAX_NOTE_PACKED_LEN); + const contentLen = reader.readField().toNumber(); const content = contentStorage.slice(0, contentLen); const noteHash = reader.readField(); From ae20fb58640edc7f309daaf9cab1bbbe0c630d08 Mon Sep 17 00:00:00 2001 From: thunkar Date: Thu, 29 May 2025 13:52:29 +0200 Subject: [PATCH 3/6] fix --- .../src/storage/note_data_provider/note_data_provider.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts b/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts index 4415918fc90d..0b6d442a3dc5 100644 --- a/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts +++ b/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts @@ -81,11 +81,11 @@ export class NoteDataProvider implements DataProvider { } async addNotes(notes: NoteDao[], scope: AztecAddress = AztecAddress.ZERO): Promise { - if (!(await this.#scopes.hasAsync(scope.toString()))) { - await this.addScope(scope); - } - return this.#store.transactionAsync(async () => { + if (!(await this.#scopes.hasAsync(scope.toString()))) { + await this.addScope(scope); + } + for (const dao of notes) { // store notes by their index in the notes hash tree // this provides the uniqueness we need to store individual notes From c1fccea125d648f31454921557acc121ca07804e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Venturo?= Date: Thu, 29 May 2025 13:01:34 +0000 Subject: [PATCH 4/6] Fix linter --- .../pxe/src/storage/note_data_provider/note_data_provider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts b/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts index 0b6d442a3dc5..d64ff78c1d73 100644 --- a/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts +++ b/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts @@ -80,7 +80,7 @@ export class NoteDataProvider implements DataProvider { return true; } - async addNotes(notes: NoteDao[], scope: AztecAddress = AztecAddress.ZERO): Promise { + addNotes(notes: NoteDao[], scope: AztecAddress = AztecAddress.ZERO): Promise { return this.#store.transactionAsync(async () => { if (!(await this.#scopes.hasAsync(scope.toString()))) { await this.addScope(scope); From 760acef4d839891ea6609991ec51ff1b7f8ec781 Mon Sep 17 00:00:00 2001 From: thunkar Date: Thu, 29 May 2025 14:15:50 +0000 Subject: [PATCH 5/6] maybe? --- yarn-project/kv-store/src/indexeddb/store.ts | 14 +++++--- .../note_data_provider/note_data_provider.ts | 34 +++++++++---------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/yarn-project/kv-store/src/indexeddb/store.ts b/yarn-project/kv-store/src/indexeddb/store.ts index 2a726bb828a2..db59880cec42 100644 --- a/yarn-project/kv-store/src/indexeddb/store.ts +++ b/yarn-project/kv-store/src/indexeddb/store.ts @@ -1,6 +1,6 @@ import type { Logger } from '@aztec/foundation/log'; -import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb'; +import { type DBSchema, type IDBPDatabase, type IDBPTransaction, deleteDB, openDB } from 'idb'; import type { AztecAsyncArray } from '../interfaces/array.js'; import type { Key, StoreSize, Value } from '../interfaces/common.js'; @@ -40,6 +40,7 @@ export interface AztecIDBSchema extends DBSchema { export class AztecIndexedDBStore implements AztecAsyncKVStore { #rootDB: IDBPDatabase; #name: string; + #currentTx: IDBPTransaction | undefined; #containers = new Set< | IndexedDBAztecArray @@ -153,15 +154,20 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore { * @returns A promise that resolves to the return value of the callback */ async transactionAsync(callback: () => Promise): Promise { - const tx = this.#rootDB.transaction('data', 'readwrite'); + // We can only have one transaction at a time for the same store + // So we need to wait for the current one to finish + if (this.#currentTx) { + await this.#currentTx.done; + } + this.#currentTx = this.#rootDB.transaction('data', 'readwrite'); for (const container of this.#containers) { - container.db = tx.store; + container.db = this.#currentTx.store; } // Avoid awaiting this promise so it doesn't get scheduled in the next microtask // By then, the tx would be closed const runningPromise = callback(); // Wait for the transaction to finish - await tx.done; + await this.#currentTx.done; for (const container of this.#containers) { container.db = undefined; } diff --git a/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts b/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts index d64ff78c1d73..ba54d0d515bb 100644 --- a/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts +++ b/yarn-project/pxe/src/storage/note_data_provider/note_data_provider.ts @@ -127,24 +127,24 @@ export class NoteDataProvider implements DataProvider { } public async unnullifyNotesAfter(blockNumber: number, synchedBlockNumber?: number): Promise { - const nullifiersToUndo: string[] = []; - const currentBlockNumber = blockNumber + 1; - const maxBlockNumber = synchedBlockNumber ?? currentBlockNumber; - for (let i = currentBlockNumber; i <= maxBlockNumber; i++) { - nullifiersToUndo.push(...(await toArray(this.#nullifiersByBlockNumber.getValuesAsync(i)))); - } - const notesIndexesToReinsert = await Promise.all( - nullifiersToUndo.map(nullifier => this.#nullifiedNotesByNullifier.getAsync(nullifier)), - ); - const notNullNoteIndexes = notesIndexesToReinsert.filter(noteIndex => noteIndex != undefined); - const nullifiedNoteBuffers = await Promise.all( - notNullNoteIndexes.map(noteIndex => this.#nullifiedNotes.getAsync(noteIndex!)), - ); - const noteDaos = nullifiedNoteBuffers - .filter(buffer => buffer != undefined) - .map(buffer => NoteDao.fromBuffer(buffer!)); - await this.#store.transactionAsync(async () => { + const nullifiersToUndo: string[] = []; + const currentBlockNumber = blockNumber + 1; + const maxBlockNumber = synchedBlockNumber ?? currentBlockNumber; + for (let i = currentBlockNumber; i <= maxBlockNumber; i++) { + nullifiersToUndo.push(...(await toArray(this.#nullifiersByBlockNumber.getValuesAsync(i)))); + } + const notesIndexesToReinsert = await Promise.all( + nullifiersToUndo.map(nullifier => this.#nullifiedNotesByNullifier.getAsync(nullifier)), + ); + const notNullNoteIndexes = notesIndexesToReinsert.filter(noteIndex => noteIndex != undefined); + const nullifiedNoteBuffers = await Promise.all( + notNullNoteIndexes.map(noteIndex => this.#nullifiedNotes.getAsync(noteIndex!)), + ); + const noteDaos = nullifiedNoteBuffers + .filter(buffer => buffer != undefined) + .map(buffer => NoteDao.fromBuffer(buffer!)); + for (const dao of noteDaos) { const noteIndex = toBufferBE(dao.index, 32).toString('hex'); await this.#notes.set(noteIndex, dao.toBuffer()); From 7c331786f80d1b57384adf967c0305fa1a5dfd14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Venturo?= Date: Thu, 29 May 2025 18:27:48 +0000 Subject: [PATCH 6/6] small renamings, add docs --- .../aztec/src/messages/processing/mod.nr | 33 ++++++---- .../processing/note_pending_validation.nr | 14 ---- .../processing/note_validation_request.nr | 64 +++++++++++++++++++ .../note_validation_request.test.ts | 43 +++++++++++++ ...lidation.ts => note_validation_request.ts} | 13 ++-- .../pxe_oracle_interface.ts | 27 ++++---- 6 files changed, 150 insertions(+), 44 deletions(-) delete mode 100644 noir-projects/aztec-nr/aztec/src/messages/processing/note_pending_validation.nr create mode 100644 noir-projects/aztec-nr/aztec/src/messages/processing/note_validation_request.nr create mode 100644 yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_validation_request.test.ts rename yarn-project/pxe/src/pxe_oracle_interface/message_processing/{note_pending_validation.ts => note_validation_request.ts} (73%) diff --git a/noir-projects/aztec-nr/aztec/src/messages/processing/mod.nr b/noir-projects/aztec-nr/aztec/src/messages/processing/mod.nr index c810f6ac1d8a..1101d57b5ad7 100644 --- a/noir-projects/aztec-nr/aztec/src/messages/processing/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/messages/processing/mod.nr @@ -1,11 +1,11 @@ pub(crate) mod pending_tagged_log; -pub(crate) mod note_pending_validation; +pub(crate) mod note_validation_request; use crate::{capsules::CapsuleArray, oracle}; use crate::messages::{ discovery::private_notes::MAX_NOTE_PACKED_LEN, processing::{ - note_pending_validation::NotePendingValidation, pending_tagged_log::PendingTaggedLog, + note_validation_request::NoteValidationRequest, pending_tagged_log::PendingTaggedLog, }, }; use protocol_types::{address::AztecAddress, hash::sha256_to_field}; @@ -14,8 +14,8 @@ use protocol_types::{address::AztecAddress, hash::sha256_to_field}; global PENDING_TAGGED_LOG_ARRAY_BASE_SLOT: Field = sha256_to_field("AZTEC_NR::PENDING_TAGGED_LOG_ARRAY_BASE_SLOT".as_bytes()); -global NOTE_PENDING_VALIDATION_ARRAY_BASE_SLOT: Field = sha256_to_field( - "AZTEC_NR::NOTE_PENDING_VALIDATION_ARRAY_BASE_SLOT".as_bytes(), +global NOTE_VALIDATION_REQUESTS_ARRAY_BASE_SLOT: Field = sha256_to_field( + "AZTEC_NR::NOTE_VALIDATION_REQUESTS_ARRAY_BASE_SLOT".as_bytes(), ); /// Searches for private logs emitted by `contract_address` that might contain messages for one of the local accounts, @@ -30,10 +30,15 @@ pub(crate) unconstrained fn get_private_logs( CapsuleArray::at(contract_address, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT) } -/// Informs PXE of a note's existence so that it can later be retrieved by the `getNotes` oracle. The note will be -/// scoped to `contract_address`, meaning other contracts will not be able to access it unless authorized. +/// Enqueues a note for validation by PXE, so that it becomes aware of a note's existence allowing for later retrieval +/// via `get_notes` oracle. The note will be scoped to `contract_address`, meaning other contracts will not be able to +/// access it unless authorized. /// -/// The packed note is what `getNotes` will later return. PXE indexes notes by `storage_slot`, so this value +/// In order for the note validation and insertion to occur, `validate_enqueued_notes` must be later called. For optimal +/// performance, accumulate as many note validation requests as possible and then validate them all at the end (which +/// results in PXE minimizing the number of network round-trips). +/// +/// The `packed_note` is what `getNotes` will later return. PXE indexes notes by `storage_slot`, so this value /// is typically used to filter notes that correspond to different state variables. `note_hash` and `nullifier` are /// the inner hashes, i.e. the raw hashes returned by `NoteHash::compute_note_hash` and /// `NoteHash::compute_nullifier`. PXE will verify that the siloed unique note hash was inserted into the tree @@ -41,8 +46,6 @@ pub(crate) unconstrained fn get_private_logs( /// /// `recipient` is the account to which the note was sent to. Other accounts will not be able to access this note (e.g. /// other accounts will not be able to see one another's token balance notes, even in the same PXE) unless authorized. -/// -/// Returns true if the note was successfully delivered and added to PXE's database. pub(crate) unconstrained fn enqueue_note_for_validation( contract_address: AztecAddress, storage_slot: Field, @@ -53,8 +56,10 @@ pub(crate) unconstrained fn enqueue_note_for_validation( tx_hash: Field, recipient: AztecAddress, ) { - CapsuleArray::at(contract_address, NOTE_PENDING_VALIDATION_ARRAY_BASE_SLOT).push( - NotePendingValidation { + // We store requests in a `CapsuleArray`, which PXE will later read from and deserialize into its version of the + // Noir `NoteValidationRequest` + CapsuleArray::at(contract_address, NOTE_VALIDATION_REQUESTS_ARRAY_BASE_SLOT).push( + NoteValidationRequest { contract_address, storage_slot, nonce, @@ -67,9 +72,13 @@ pub(crate) unconstrained fn enqueue_note_for_validation( ) } +/// Validates all note validation requests enqueued via `enqueue_note_for_validation`, inserting them into the note +/// database and making them queryable via `get_notes`. +/// +/// This automatically clears the validation request queue, so no further work needs to be done by the caller. pub(crate) unconstrained fn validate_enqueued_notes(contract_address: AztecAddress) { oracle::message_processing::validate_enqueued_notes( contract_address, - NOTE_PENDING_VALIDATION_ARRAY_BASE_SLOT, + NOTE_VALIDATION_REQUESTS_ARRAY_BASE_SLOT, ); } diff --git a/noir-projects/aztec-nr/aztec/src/messages/processing/note_pending_validation.nr b/noir-projects/aztec-nr/aztec/src/messages/processing/note_pending_validation.nr deleted file mode 100644 index 6c31910b07dd..000000000000 --- a/noir-projects/aztec-nr/aztec/src/messages/processing/note_pending_validation.nr +++ /dev/null @@ -1,14 +0,0 @@ -use crate::messages::discovery::private_notes::MAX_NOTE_PACKED_LEN; -use protocol_types::{address::AztecAddress, traits::Serialize}; - -#[derive(Serialize)] -pub(crate) struct NotePendingValidation { - pub contract_address: AztecAddress, - pub storage_slot: Field, - pub nonce: Field, - pub packed_note: BoundedVec, - pub note_hash: Field, - pub nullifier: Field, - pub tx_hash: Field, - pub recipient: AztecAddress, -} diff --git a/noir-projects/aztec-nr/aztec/src/messages/processing/note_validation_request.nr b/noir-projects/aztec-nr/aztec/src/messages/processing/note_validation_request.nr new file mode 100644 index 000000000000..47b1cb25351a --- /dev/null +++ b/noir-projects/aztec-nr/aztec/src/messages/processing/note_validation_request.nr @@ -0,0 +1,64 @@ +use crate::messages::discovery::private_notes::MAX_NOTE_PACKED_LEN; +use protocol_types::{address::AztecAddress, traits::Serialize}; + +/// Intermediate struct used to perform batch note validation by PXE. The `validateEnqueuedNotes` oracle expects for +/// values of this type to be stored in a `CapsuleArray`. +#[derive(Serialize)] +pub(crate) struct NoteValidationRequest { + pub contract_address: AztecAddress, + pub storage_slot: Field, + pub nonce: Field, + pub packed_note: BoundedVec, + pub note_hash: Field, + pub nullifier: Field, + pub tx_hash: Field, + pub recipient: AztecAddress, +} + +mod test { + use super::NoteValidationRequest; + use protocol_types::{address::AztecAddress, traits::{FromField, Serialize}}; + + #[test] + fn serialization_matches_typescript() { + let request = NoteValidationRequest { + contract_address: AztecAddress::from_field(1), + storage_slot: 2, + nonce: 3, + packed_note: BoundedVec::from_array([4, 5]), + note_hash: 6, + nullifier: 7, + tx_hash: 8, + recipient: AztecAddress::from_field(9), + }; + + // We define the serialization in Noir and the deserialization in TS. If the deserialization changes from the + // snapshot value below, then note_validation_request.test.ts must be updated with the same value. + // Ideally we'd autogenerate this, but for now we only have single-sided snapshot generation, from TS to Noir, + // which is not what we need here. + let expected_serialization = [ + 0x0000000000000000000000000000000000000000000000000000000000000001, + 0x0000000000000000000000000000000000000000000000000000000000000002, + 0x0000000000000000000000000000000000000000000000000000000000000003, + 0x0000000000000000000000000000000000000000000000000000000000000004, + 0x0000000000000000000000000000000000000000000000000000000000000005, + 0x0000000000000000000000000000000000000000000000000000000000000000, + 0x0000000000000000000000000000000000000000000000000000000000000000, + 0x0000000000000000000000000000000000000000000000000000000000000000, + 0x0000000000000000000000000000000000000000000000000000000000000000, + 0x0000000000000000000000000000000000000000000000000000000000000000, + 0x0000000000000000000000000000000000000000000000000000000000000000, + 0x0000000000000000000000000000000000000000000000000000000000000000, + 0x0000000000000000000000000000000000000000000000000000000000000000, + 0x0000000000000000000000000000000000000000000000000000000000000000, + 0x0000000000000000000000000000000000000000000000000000000000000000, + 0x0000000000000000000000000000000000000000000000000000000000000002, + 0x0000000000000000000000000000000000000000000000000000000000000006, + 0x0000000000000000000000000000000000000000000000000000000000000007, + 0x0000000000000000000000000000000000000000000000000000000000000008, + 0x0000000000000000000000000000000000000000000000000000000000000009, + ]; + + assert_eq(request.serialize(), expected_serialization); + } +} diff --git a/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_validation_request.test.ts b/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_validation_request.test.ts new file mode 100644 index 000000000000..5b1007b2df17 --- /dev/null +++ b/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_validation_request.test.ts @@ -0,0 +1,43 @@ +import { Fr } from '@aztec/foundation/fields'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; +import { TxHash } from '@aztec/stdlib/tx'; + +import { NoteValidationRequest } from './note_validation_request.js'; + +describe('NoteValidationRequest', () => { + it('output of Noir serialization deserializes as expected', () => { + const serialized = [ + '0x0000000000000000000000000000000000000000000000000000000000000001', + '0x0000000000000000000000000000000000000000000000000000000000000002', + '0x0000000000000000000000000000000000000000000000000000000000000003', + '0x0000000000000000000000000000000000000000000000000000000000000004', + '0x0000000000000000000000000000000000000000000000000000000000000005', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000002', + '0x0000000000000000000000000000000000000000000000000000000000000006', + '0x0000000000000000000000000000000000000000000000000000000000000007', + '0x0000000000000000000000000000000000000000000000000000000000000008', + '0x0000000000000000000000000000000000000000000000000000000000000009', + ].map(Fr.fromHexString); + + const request = NoteValidationRequest.fromFields(serialized); + + expect(request.contractAddress).toEqual(AztecAddress.fromBigInt(1n)); + expect(request.storageSlot).toEqual(new Fr(2)); + expect(request.nonce).toEqual(new Fr(3)); + expect(request.content).toEqual([new Fr(4), new Fr(5)]); + expect(request.noteHash).toEqual(new Fr(6)); + expect(request.nullifier).toEqual(new Fr(7)); + expect(request.txHash).toEqual(TxHash.fromBigInt(8n)); + expect(request.recipient).toEqual(AztecAddress.fromBigInt(9n)); + }); +}); diff --git a/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts b/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_validation_request.ts similarity index 73% rename from yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts rename to yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_validation_request.ts index c6c0d41ac28b..208a4493b066 100644 --- a/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_pending_validation.ts +++ b/yarn-project/pxe/src/pxe_oracle_interface/message_processing/note_validation_request.ts @@ -3,13 +3,14 @@ import { FieldReader } from '@aztec/foundation/serialize'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { TxHash } from '@aztec/stdlib/tx'; -const MAX_NOTE_PACKED_LEN = 12; // TODO ?? +// TODO(#14617): should we compute this from constants? This value is aztec-nr specific. +const MAX_NOTE_PACKED_LEN = 12; /** - * Represents a pending tagged log as it is stored in the pending tagged log array to which the fetchTaggedLogs oracle - * inserts found private logs. A TS version of `pending_tagged_log.nr`. + * Intermediate struct used to perform batch note validation by PXE. The `validateEnqueuedNotes` oracle expects for + * values of this type to be stored in a `CapsuleArray`. */ -export class NotePendingValidation { +export class NoteValidationRequest { constructor( public contractAddress: AztecAddress, public storageSlot: Fr, @@ -21,7 +22,7 @@ export class NotePendingValidation { public recipient: AztecAddress, ) {} - static fromFields(fields: Fr[] | FieldReader): NotePendingValidation { + static fromFields(fields: Fr[] | FieldReader): NoteValidationRequest { const reader = FieldReader.asReader(fields); const contractAddress = AztecAddress.fromField(reader.readField()); @@ -37,7 +38,7 @@ export class NotePendingValidation { const txHash = TxHash.fromField(reader.readField()); const recipient = AztecAddress.fromField(reader.readField()); - return new NotePendingValidation( + return new NoteValidationRequest( contractAddress, storageSlot, nonce, diff --git a/yarn-project/pxe/src/pxe_oracle_interface/pxe_oracle_interface.ts b/yarn-project/pxe/src/pxe_oracle_interface/pxe_oracle_interface.ts index e7100dd889a6..fe8455f78e3d 100644 --- a/yarn-project/pxe/src/pxe_oracle_interface/pxe_oracle_interface.ts +++ b/yarn-project/pxe/src/pxe_oracle_interface/pxe_oracle_interface.ts @@ -39,7 +39,7 @@ import type { NoteDataProvider } from '../storage/note_data_provider/note_data_p import type { PrivateEventDataProvider } from '../storage/private_event_data_provider/private_event_data_provider.js'; import type { SyncDataProvider } from '../storage/sync_data_provider/sync_data_provider.js'; import type { TaggingDataProvider } from '../storage/tagging_data_provider/tagging_data_provider.js'; -import { NotePendingValidation } from './message_processing/note_pending_validation.js'; +import { NoteValidationRequest } from './message_processing/note_validation_request.js'; import { WINDOW_HALF_SIZE, getIndexedTaggingSecretsForTheWindow, getInitialIndexesMap } from './tagging_utils.js'; /** @@ -613,25 +613,28 @@ export class PXEOracleInterface implements ExecutionDataProvider { contractAddress: AztecAddress, notePendingValidationArrayBaseSlot: Fr, ): Promise { - const notesPendingValidation = ( + // We read all note validation requests and process them all concurrently. This makes the process much faster as we + // don't need to wait for the network round-trip. + const noteValidationRequests = ( await this.capsuleDataProvider.readCapsuleArray(contractAddress, notePendingValidationArrayBaseSlot) - ).map(NotePendingValidation.fromFields); + ).map(NoteValidationRequest.fromFields); await Promise.all( - notesPendingValidation.map(note => + noteValidationRequests.map(request => this.deliverNote( - note.contractAddress, - note.storageSlot, - note.nonce, - note.content, - note.noteHash, - note.nullifier, - note.txHash, - note.recipient, + request.contractAddress, + request.storageSlot, + request.nonce, + request.content, + request.noteHash, + request.nullifier, + request.txHash, + request.recipient, ), ), ); + // Requests are cleared once we're done. await this.capsuleDataProvider.resetCapsuleArray(contractAddress, notePendingValidationArrayBaseSlot, []); }