diff --git a/docs/docs/migration_notes.md b/docs/docs/migration_notes.md index 31585e136f5b..0b1baa4bb3f4 100644 --- a/docs/docs/migration_notes.md +++ b/docs/docs/migration_notes.md @@ -9,6 +9,10 @@ Aztec is in full-speed development. Literally every version breaks compatibility ## TBD +## [Aztec node] + +Network config. The node now pulls default configuration from the public repository [AztecProtocol/networks](https://github.com/AztecProtocol/networks) after it applies the configuration it takes from the running environment and the configuration values baked into the source code. See associated [Design document](https://github.com/AztecProtocol/engineering-designs/blob/15415a62a7c8e901acb8e523625e91fc6f71dce4/docs/network-config/dd.md) + ## [Aztec Tools] ### Contract compilation now requires two steps diff --git a/yarn-project/aztec/src/cli/aztec_start_options.ts b/yarn-project/aztec/src/cli/aztec_start_options.ts index e5f06b1d79f2..f4f26ca82e99 100644 --- a/yarn-project/aztec/src/cli/aztec_start_options.ts +++ b/yarn-project/aztec/src/cli/aztec_start_options.ts @@ -106,7 +106,7 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = { configToFlag('--auto-update-url', sharedNodeConfigMappings.autoUpdateUrl), configToFlag('--sync-mode', sharedNodeConfigMappings.syncMode), - configToFlag('--snapshots-url', sharedNodeConfigMappings.snapshotsUrl), + configToFlag('--snapshots-urls', sharedNodeConfigMappings.snapshotsUrls), ], SANDBOX: [ { diff --git a/yarn-project/cli/src/config/chain_l2_config.ts b/yarn-project/cli/src/config/chain_l2_config.ts index be66b9490d4b..a0109c01f432 100644 --- a/yarn-project/cli/src/config/chain_l2_config.ts +++ b/yarn-project/cli/src/config/chain_l2_config.ts @@ -25,7 +25,7 @@ export type L2ChainConfig = L1ContractsConfig & seqMinTxsPerBlock: number; seqMaxTxsPerBlock: number; realProofs: boolean; - snapshotsUrl: string; + snapshotsUrls: string[]; autoUpdate: SharedNodeConfig['autoUpdate']; autoUpdateUrl?: string; maxTxPoolSize: number; @@ -99,7 +99,7 @@ export const stagingIgnitionL2ChainConfig: L2ChainConfig = { seqMinTxsPerBlock: 0, seqMaxTxsPerBlock: 0, realProofs: true, - snapshotsUrl: 'https://storage.googleapis.com/aztec-testnet/snapshots/staging-ignition/', + snapshotsUrls: ['https://storage.googleapis.com/aztec-testnet/snapshots/staging-ignition/'], autoUpdate: 'config-and-version', autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/staging-ignition.json', maxTxPoolSize: 100_000_000, // 100MB @@ -180,7 +180,7 @@ export const stagingPublicL2ChainConfig: L2ChainConfig = { seqMinTxsPerBlock: 0, seqMaxTxsPerBlock: 20, realProofs: true, - snapshotsUrl: 'https://storage.googleapis.com/aztec-testnet/snapshots/staging-public/', + snapshotsUrls: ['https://storage.googleapis.com/aztec-testnet/snapshots/staging-public/'], autoUpdate: 'config-and-version', autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/staging-public.json', publicIncludeMetrics, @@ -233,7 +233,7 @@ export const testnetL2ChainConfig: L2ChainConfig = { seqMinTxsPerBlock: 0, seqMaxTxsPerBlock: 20, realProofs: true, - snapshotsUrl: 'https://storage.googleapis.com/aztec-testnet/snapshots/testnet/', + snapshotsUrls: ['https://storage.googleapis.com/aztec-testnet/snapshots/testnet/'], autoUpdate: 'config-and-version', autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/testnet.json', maxTxPoolSize: 100_000_000, // 100MB @@ -289,7 +289,7 @@ export const ignitionL2ChainConfig: L2ChainConfig = { seqMinTxsPerBlock: 0, seqMaxTxsPerBlock: 0, realProofs: true, - snapshotsUrl: 'https://storage.googleapis.com/aztec-testnet/snapshots/ignition/', + snapshotsUrls: ['https://storage.googleapis.com/aztec-testnet/snapshots/ignition/'], autoUpdate: 'notify', autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/ignition.json', maxTxPoolSize: 100_000_000, // 100MB @@ -424,7 +424,7 @@ export async function enrichEnvironmentWithChainConfig(networkName: NetworkNames enrichVar('SEQ_MAX_TX_PER_BLOCK', config.seqMaxTxsPerBlock.toString()); enrichVar('PROVER_REAL_PROOFS', config.realProofs.toString()); enrichVar('PXE_PROVER_ENABLED', config.realProofs.toString()); - enrichVar('SYNC_SNAPSHOTS_URL', config.snapshotsUrl); + enrichVar('SYNC_SNAPSHOTS_URLS', config.snapshotsUrls.join(',')); enrichVar('P2P_MAX_TX_POOL_SIZE', config.maxTxPoolSize.toString()); enrichVar('DATA_STORE_MAP_SIZE_KB', config.dbMapSizeKb.toString()); diff --git a/yarn-project/cli/src/config/network_config.ts b/yarn-project/cli/src/config/network_config.ts index f92edb51a394..2c04ff1d3656 100644 --- a/yarn-project/cli/src/config/network_config.ts +++ b/yarn-project/cli/src/config/network_config.ts @@ -93,7 +93,7 @@ export async function enrichEnvironmentWithNetworkConfig(networkName: NetworkNam enrichVar('BOOTSTRAP_NODES', networkConfig.bootnodes.join(',')); enrichVar('L1_CHAIN_ID', String(networkConfig.l1ChainId)); - enrichVar('SYNC_SNAPSHOTS_URL', networkConfig.snapshots.join(',')); + enrichVar('SYNC_SNAPSHOTS_URLS', networkConfig.snapshots.join(',')); enrichEthAddressVar('REGISTRY_CONTRACT_ADDRESS', networkConfig.registryAddress.toString()); if (networkConfig.feeAssetHandlerAddress) { diff --git a/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts b/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts index eb82a8365d0e..07b9395ca0cb 100644 --- a/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts +++ b/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts @@ -4,11 +4,11 @@ import { RollupContract } from '@aztec/ethereum'; import { ChainMonitor } from '@aztec/ethereum/test'; import { randomBytes } from '@aztec/foundation/crypto'; import { tryRmDir } from '@aztec/foundation/fs'; -import { withLogNameSuffix } from '@aztec/foundation/log'; +import { logger, withLogNameSuffix } from '@aztec/foundation/log'; import { bufferToHex } from '@aztec/foundation/string'; import { ProverNode, type ProverNodeConfig } from '@aztec/prover-node'; -import { mkdtemp, readdir } from 'fs/promises'; +import { cp, mkdtemp, readFile, readdir, rm, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -24,6 +24,8 @@ describe('e2e_snapshot_sync', () => { let snapshotDir: string; let snapshotLocation: string; + let cleanupDirs: string[]; + beforeAll(async () => { context = await setup(0, { minTxsPerBlock: 0, @@ -38,6 +40,7 @@ describe('e2e_snapshot_sync', () => { log = context.logger; snapshotDir = await mkdtemp(join(tmpdir(), 'snapshots-')); + cleanupDirs = [snapshotDir]; snapshotLocation = `file://${snapshotDir}`; monitor = new ChainMonitor(RollupContract.getFromConfig(context.config), context.dateProvider, log).start(); }); @@ -45,7 +48,7 @@ describe('e2e_snapshot_sync', () => { afterAll(async () => { await monitor.stop(); await context.teardown(); - await tryRmDir(snapshotDir, log); + await Promise.all(cleanupDirs.map(dir => tryRmDir(dir, log))); }); // Adapted from epochs-test @@ -95,7 +98,7 @@ describe('e2e_snapshot_sync', () => { it('downloads snapshot when syncing new node', async () => { log.warn(`Syncing brand new node with snapshot sync`); - const node = await createNonValidatorNode('1', { snapshotsUrl: snapshotLocation, syncMode: 'snapshot' }); + const node = await createNonValidatorNode('1', { snapshotsUrls: [snapshotLocation], syncMode: 'snapshot' }); log.warn(`New node synced`); await expectNodeSyncedToL2Block(node, L2_TARGET_BLOCK_NUM); @@ -116,7 +119,7 @@ describe('e2e_snapshot_sync', () => { it('downloads snapshot when syncing new prover node', async () => { log.warn(`Syncing brand new prover node with snapshot sync`); - const node = await createTestProverNode({ snapshotsUrl: snapshotLocation, syncMode: 'snapshot' }); + const node = await createTestProverNode({ snapshotsUrls: [snapshotLocation], syncMode: 'snapshot' }); log.warn(`New node prover synced`); await expectNodeSyncedToL2Block(node, L2_TARGET_BLOCK_NUM); @@ -124,4 +127,108 @@ describe('e2e_snapshot_sync', () => { log.warn(`Stopping new prover node`); await node.stop(); }); + + it('downloads snapshot from multiple sources', async () => { + log.warn(`Setting up multiple snapshot locations with different L1 block heights`); + + // Create two additional snapshot directories (third one is the existing snapshotDir) + const snapshotDir1 = await mkdtemp(join(tmpdir(), 'snapshots-1-')); + const snapshotDir2 = await mkdtemp(join(tmpdir(), 'snapshots-2-')); + const snapshotLocation1 = `file://${snapshotDir1}`; + const snapshotLocation2 = `file://${snapshotDir2}`; + const snapshotLocation3 = snapshotLocation; // Use the existing snapshot + + cleanupDirs.push(snapshotDir1, snapshotDir2); + + // Copy the existing snapshot to snapshot 1 and 2 + log.warn(`Copying existing snapshot to two new locations`); + const originalFiles = await readdir(snapshotDir, { recursive: true }); + log.warn(`Found ${originalFiles.length} files in snapshot directory`); + + // Find the index.json file + const indexFile = originalFiles.find(f => typeof f === 'string' && f.includes('index.json')); + expect(indexFile).toBeDefined(); + + // Copy all files recursively + for (const file of originalFiles) { + const srcPath = join(snapshotDir, file as string); + const destPath1 = join(snapshotDir1, file as string); + const destPath2 = join(snapshotDir2, file as string); + + try { + await cp(srcPath, destPath1, { recursive: true }); + await cp(srcPath, destPath2, { recursive: true }); + } catch { + // Skip if it's a directory or already copied + } + } + + // Update index jsons + for (const newDir of [snapshotDir1, snapshotDir2]) { + const files = await readdir(newDir, { recursive: true }); + const indexFile = files.find(f => typeof f === 'string' && f.includes('index.json')); + expect(indexFile).toBeDefined(); + const indexContents = await readFile(join(newDir, indexFile!), 'utf-8'); + const updatedContents = indexContents.replaceAll(snapshotDir, newDir); + await writeFile(join(newDir, indexFile!), updatedContents); + logger.info(`Updated index file in ${newDir}`, { updatedContents }); + } + + // Read the original index.json to get the base L1 block number + const indexPath3 = join(snapshotDir, indexFile!); + const indexContent = JSON.parse(await readFile(indexPath3, 'utf-8')); + const baseL1Block = indexContent.snapshots[0].l1BlockNumber; + log.warn(`Base L1 block number: ${baseL1Block}`); + + // Modify snapshot 1: increase L1 block height (highest) and corrupt it + log.warn(`Modifying snapshot 1 to have highest L1 block height`); + const indexPath1 = join(snapshotDir1, indexFile!); + const index1 = JSON.parse(await readFile(indexPath1, 'utf-8')); + index1.snapshots[0].l1BlockNumber = baseL1Block + 200; // Highest + await writeFile(indexPath1, JSON.stringify(index1, null, 2)); + + // Corrupt snapshot 1 by removing one of the database files + log.warn(`Corrupting snapshot 1 by removing a database file`); + const snapshot1Files = await readdir(snapshotDir1, { recursive: true }); + const dbFile = snapshot1Files.find(f => typeof f === 'string' && f.endsWith('.db')); + expect(dbFile).toBeDefined(); + await rm(join(snapshotDir1, dbFile!)); + log.warn(`Removed ${dbFile} from snapshot 1`); + + // Modify snapshot 2: decrease L1 block height (lowest) + log.warn(`Modifying snapshot 2 to have lowest L1 block height`); + const indexPath2 = join(snapshotDir2, indexFile!); + const index2 = JSON.parse(await readFile(indexPath2, 'utf-8')); + index2.snapshots[0].l1BlockNumber = baseL1Block - 1; // Lowest + await writeFile(indexPath2, JSON.stringify(index2, null, 2)); + + // Snapshot 3 (original) has the middle L1 block height (baseL1Block) + log.warn(`Snapshot 3 (original) has L1 block height ${baseL1Block} (middle)`); + + // Now sync a new node with all three URLs + // Snapshot 1: highest L1 block (baseL1Block + 200) but corrupted (should fail) + // Snapshot 2: lowest L1 block (baseL1Block - 1) but valid + // Snapshot 3: middle L1 block (baseL1Block) and valid (should be selected after 1 fails) + log.warn(`Syncing brand new node with three snapshot URLs`); + const node = await createNonValidatorNode('multi-url', { + snapshotsUrls: [snapshotLocation1, snapshotLocation2, snapshotLocation3], + syncMode: 'snapshot', + }); + + log.warn(`New node synced with fallback logic`); + await expectNodeSyncedToL2Block(node, L2_TARGET_BLOCK_NUM); + + const block = await node.getBlock(L2_TARGET_BLOCK_NUM); + expect(block).toBeDefined(); + const blockHash = await block!.hash(); + + log.warn(`Checking for L2 block ${L2_TARGET_BLOCK_NUM} with hash ${blockHash} on both nodes`); + const getBlockHashLeafIndex = (node: AztecNode) => + node.findLeavesIndexes(L2_TARGET_BLOCK_NUM, MerkleTreeId.ARCHIVE, [blockHash]).then(([i]) => i); + expect(await getBlockHashLeafIndex(context.aztecNode)).toBeDefined(); + expect(await getBlockHashLeafIndex(node)).toBeDefined(); + + log.warn(`Stopping new node`); + await node.stop(); + }); }); diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 52af57bdf10b..5906049f2bd9 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -151,6 +151,7 @@ export type EnvVar = | 'PROVER_COORDINATION_NODE_URLS' | 'PROVER_FAILED_PROOF_STORE' | 'PROVER_NODE_FAILED_EPOCH_STORE' + | 'PROVER_NODE_DISABLE_PROOF_PUBLISH' | 'PROVER_ID' | 'PROVER_NODE_POLLING_INTERVAL_MS' | 'PROVER_NODE_MAX_PENDING_JOBS' @@ -164,7 +165,6 @@ export type EnvVar = | 'PROVER_PUBLISHER_PRIVATE_KEYS' | 'PROVER_PUBLISHER_ADDRESSES' | 'PROVER_PUBLISHER_ALLOW_INVALID_STATES' - | 'PROVER_PUBLISHER_ENABLED' | 'PROVER_REAL_PROOFS' | 'PROVER_TEST_DELAY_FACTOR' | 'PROVER_TEST_DELAY_MS' @@ -188,7 +188,6 @@ export type EnvVar = | 'SEQ_PUBLISHER_PRIVATE_KEYS' | 'SEQ_PUBLISHER_ADDRESSES' | 'SEQ_PUBLISHER_ALLOW_INVALID_STATES' - | 'SEQ_PUBLISHER_ENABLED' | 'SEQ_TX_POLLING_INTERVAL_MS' | 'SEQ_ENFORCE_TIME_TABLE' | 'SEQ_MAX_L1_TX_INCLUSION_TIME_INTO_SLOT' @@ -214,6 +213,7 @@ export type EnvVar = | 'SLASH_MAX_PAYLOAD_SIZE' | 'SLASH_EXECUTE_ROUNDS_LOOK_BACK' | 'SYNC_MODE' + | 'SYNC_SNAPSHOTS_URLS' | 'SYNC_SNAPSHOTS_URL' | 'TELEMETRY' | 'TEST_ACCOUNTS' diff --git a/yarn-project/node-lib/src/actions/snapshot-sync.ts b/yarn-project/node-lib/src/actions/snapshot-sync.ts index ba2d61c4c834..6c81fb18c52e 100644 --- a/yarn-project/node-lib/src/actions/snapshot-sync.ts +++ b/yarn-project/node-lib/src/actions/snapshot-sync.ts @@ -26,11 +26,12 @@ import type { SharedNodeConfig } from '../config/index.js'; // Half day worth of L1 blocks const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12; -type SnapshotSyncConfig = Pick & +type SnapshotSyncConfig = Pick & Pick & Pick & Required & EthereumClientConfig & { + snapshotsUrls?: string[]; minL1BlocksToTriggerReplace?: number; }; @@ -39,14 +40,14 @@ type SnapshotSyncConfig = Pick & * Behaviour depends on syncing mode. */ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) { - const { syncMode, snapshotsUrl, dataDirectory, l1ChainId, rollupVersion, l1Contracts } = config; + const { syncMode, snapshotsUrls, dataDirectory, l1ChainId, rollupVersion, l1Contracts } = config; if (syncMode === 'full') { log.debug('Snapshot sync is disabled. Running full sync.', { syncMode: syncMode }); return false; } - if (!snapshotsUrl) { - log.verbose('Snapshot sync is disabled. No snapshots URL provided.'); + if (!snapshotsUrls || snapshotsUrls.length === 0) { + log.verbose('Snapshot sync is disabled. No snapshots URLs provided.'); return false; } @@ -55,15 +56,7 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) { return false; } - let fileStore: ReadOnlyFileStore; - try { - fileStore = await createReadOnlyFileStore(snapshotsUrl, log); - } catch (err) { - log.error(`Invalid config for downloading snapshots`, err); - return false; - } - - // Create an archiver store to check the current state + // Create an archiver store to check the current state (do this only once) log.verbose(`Creating temporary archiver data store`); const archiverStore = await createArchiverStore(config); let archiverL1BlockNumber: bigint | undefined; @@ -102,65 +95,111 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) { rollupVersion, rollupAddress: l1Contracts.rollupAddress, }; - let snapshot: SnapshotMetadata | undefined; - try { - snapshot = await getLatestSnapshotMetadata(indexMetadata, fileStore); - } catch (err) { - log.error(`Failed to get latest snapshot metadata. Skipping snapshot sync.`, err, { - ...indexMetadata, - snapshotsUrl, - }); - return false; - } - if (!snapshot) { - log.verbose(`No snapshot found. Skipping snapshot sync.`, { ...indexMetadata, snapshotsUrl }); - return false; - } + // Fetch latest snapshot from each URL + type SnapshotCandidate = { snapshot: SnapshotMetadata; url: string; fileStore: ReadOnlyFileStore }; + const snapshotCandidates: SnapshotCandidate[] = []; - if (snapshot.schemaVersions.archiver !== ARCHIVER_DB_VERSION) { - log.warn( - `Skipping snapshot sync as last snapshot has schema version ${snapshot.schemaVersions.archiver} but expected ${ARCHIVER_DB_VERSION}.`, - snapshot, - ); - return false; - } + for (const snapshotsUrl of snapshotsUrls) { + let fileStore: ReadOnlyFileStore; + try { + fileStore = await createReadOnlyFileStore(snapshotsUrl, log); + } catch (err) { + log.error(`Invalid config for downloading snapshots from ${snapshotsUrl}`, err); + continue; + } - if (snapshot.schemaVersions.worldState !== WORLD_STATE_DB_VERSION) { - log.warn( - `Skipping snapshot sync as last snapshot has world state schema version ${snapshot.schemaVersions.worldState} but we expected ${WORLD_STATE_DB_VERSION}.`, - snapshot, - ); - return false; + let snapshot: SnapshotMetadata | undefined; + try { + snapshot = await getLatestSnapshotMetadata(indexMetadata, fileStore); + } catch (err) { + log.error(`Failed to get latest snapshot metadata from ${snapshotsUrl}. Skipping this URL.`, err, { + ...indexMetadata, + snapshotsUrl, + }); + continue; + } + + if (!snapshot) { + log.verbose(`No snapshot found at ${snapshotsUrl}. Skipping this URL.`, { ...indexMetadata, snapshotsUrl }); + continue; + } + + if (snapshot.schemaVersions.archiver !== ARCHIVER_DB_VERSION) { + log.warn( + `Skipping snapshot from ${snapshotsUrl} as it has schema version ${snapshot.schemaVersions.archiver} but expected ${ARCHIVER_DB_VERSION}.`, + snapshot, + ); + continue; + } + + if (snapshot.schemaVersions.worldState !== WORLD_STATE_DB_VERSION) { + log.warn( + `Skipping snapshot from ${snapshotsUrl} as it has world state schema version ${snapshot.schemaVersions.worldState} but we expected ${WORLD_STATE_DB_VERSION}.`, + snapshot, + ); + continue; + } + + if (archiverL1BlockNumber && snapshot.l1BlockNumber < archiverL1BlockNumber) { + log.verbose( + `Skipping snapshot from ${snapshotsUrl} since local archiver is at L1 block ${archiverL1BlockNumber} which is further than snapshot at ${snapshot.l1BlockNumber}`, + { snapshot, archiverL1BlockNumber, snapshotsUrl }, + ); + continue; + } + + if (archiverL1BlockNumber && snapshot.l1BlockNumber - Number(archiverL1BlockNumber) < minL1BlocksToTriggerReplace) { + log.verbose( + `Skipping snapshot from ${snapshotsUrl} as archiver is less than ${ + snapshot.l1BlockNumber - Number(archiverL1BlockNumber) + } L1 blocks behind this snapshot.`, + { snapshot, archiverL1BlockNumber, snapshotsUrl }, + ); + continue; + } + + snapshotCandidates.push({ snapshot, url: snapshotsUrl, fileStore }); } - if (archiverL1BlockNumber && snapshot.l1BlockNumber < archiverL1BlockNumber) { - log.verbose( - `Skipping snapshot sync since local archiver is at L1 block ${archiverL1BlockNumber} which is further than last snapshot at ${snapshot.l1BlockNumber}`, - { snapshot, archiverL1BlockNumber }, - ); + if (snapshotCandidates.length === 0) { + log.verbose(`No valid snapshots found from any URL. Skipping snapshot sync.`, { ...indexMetadata, snapshotsUrls }); return false; } - if (archiverL1BlockNumber && snapshot.l1BlockNumber - Number(archiverL1BlockNumber) < minL1BlocksToTriggerReplace) { - log.verbose( - `Skipping snapshot sync as archiver is less than ${ - snapshot.l1BlockNumber - Number(archiverL1BlockNumber) - } L1 blocks behind latest snapshot.`, - { snapshot, archiverL1BlockNumber }, - ); - return false; + // Sort candidates by L1 block number (highest first) + snapshotCandidates.sort((a, b) => b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber); + + // Try each candidate in order until one succeeds + for (const { snapshot, url } of snapshotCandidates) { + const { l1BlockNumber, l2BlockNumber } = snapshot; + log.info(`Attempting to sync from snapshot at L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, { + snapshot, + snapshotsUrl: url, + }); + + try { + await snapshotSync(snapshot, log, { + dataDirectory: config.dataDirectory!, + rollupAddress: config.l1Contracts.rollupAddress, + snapshotsUrl: url, + }); + log.info(`Snapshot synced to L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, { + snapshot, + snapshotsUrl: url, + }); + return true; + } catch (err) { + log.error(`Failed to download snapshot from ${url}. Trying next candidate.`, err, { + snapshot, + snapshotsUrl: url, + }); + continue; + } } - const { l1BlockNumber, l2BlockNumber } = snapshot; - log.info(`Syncing from snapshot at L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, { snapshot, snapshotsUrl }); - await snapshotSync(snapshot, log, { - dataDirectory: config.dataDirectory!, - rollupAddress: config.l1Contracts.rollupAddress, - snapshotsUrl, - }); - log.info(`Snapshot synced to L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, { snapshot }); - return true; + log.error(`Failed to download snapshot from all URLs.`, { snapshotsUrls }); + return false; } /** diff --git a/yarn-project/node-lib/src/config/index.ts b/yarn-project/node-lib/src/config/index.ts index e3ce5d56005f..f59f54ab7f05 100644 --- a/yarn-project/node-lib/src/config/index.ts +++ b/yarn-project/node-lib/src/config/index.ts @@ -7,8 +7,8 @@ export type SharedNodeConfig = { sponsoredFPC: boolean; /** Sync mode: full to always sync via L1, snapshot to download a snapshot if there is no local data, force-snapshot to download even if there is local data. */ syncMode: 'full' | 'snapshot' | 'force-snapshot'; - /** Base URL for snapshots index. Index file will be searched at `SNAPSHOTS_BASE_URL/aztec-L1_CHAIN_ID-VERSION-ROLLUP_ADDRESS/index.json` */ - snapshotsUrl?: string; + /** Base URLs for snapshots index. Index file will be searched at `SNAPSHOTS_BASE_URL/aztec-L1_CHAIN_ID-VERSION-ROLLUP_ADDRESS/index.json` */ + snapshotsUrls?: string[]; /** Auto update mode: disabled - to completely ignore remote signals to update the node. enabled - to respect the signals (potentially shutting this node down). log - check for updates but log a warning instead of applying them*/ autoUpdate?: 'disabled' | 'notify' | 'config' | 'config-and-version'; @@ -36,9 +36,16 @@ export const sharedNodeConfigMappings: ConfigMappingsType = { 'Set sync mode to `full` to always sync via L1, `snapshot` to download a snapshot if there is no local data, `force-snapshot` to download even if there is local data.', defaultValue: 'snapshot', }, - snapshotsUrl: { - env: 'SYNC_SNAPSHOTS_URL', - description: 'Base URL for snapshots index.', + snapshotsUrls: { + env: 'SYNC_SNAPSHOTS_URLS', + description: 'Base URLs for snapshots index, comma-separated.', + parseEnv: (val: string) => + val + .split(',') + .map(url => url.trim()) + .filter(url => url.length > 0), + fallback: ['SYNC_SNAPSHOTS_URL'], + defaultValue: [], }, autoUpdate: { env: 'AUTO_UPDATE', diff --git a/yarn-project/prover-node/src/config.ts b/yarn-project/prover-node/src/config.ts index b4d54cfe644d..a2f468006dc2 100644 --- a/yarn-project/prover-node/src/config.ts +++ b/yarn-project/prover-node/src/config.ts @@ -1,7 +1,12 @@ import { type ArchiverConfig, archiverConfigMappings } from '@aztec/archiver/config'; import type { ACVMConfig, BBConfig } from '@aztec/bb-prover/config'; import { type GenesisStateConfig, genesisStateConfigMappings } from '@aztec/ethereum'; -import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config'; +import { + type ConfigMappingsType, + booleanConfigHelper, + getConfigFromMappings, + numberConfigHelper, +} from '@aztec/foundation/config'; import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config'; import { type KeyStore, type KeyStoreConfig, ethPrivateKeySchema, keyStoreConfigMappings } from '@aztec/node-keystore'; import { type SharedNodeConfig, sharedNodeConfigMappings } from '@aztec/node-lib/config'; @@ -38,6 +43,7 @@ export type SpecificProverNodeConfig = { proverNodePollingIntervalMs: number; proverNodeMaxParallelBlocksPerEpoch: number; proverNodeFailedEpochStore: string | undefined; + proverNodeDisableProofPublish?: boolean; txGatheringTimeoutMs: number; txGatheringIntervalMs: number; txGatheringBatchSize: number; @@ -85,6 +91,11 @@ const specificProverNodeConfigMappings: ConfigMappingsType = { diff --git a/yarn-project/prover-node/src/job/epoch-proving-job.test.ts b/yarn-project/prover-node/src/job/epoch-proving-job.test.ts index f6e39b72e075..fb6462737f18 100644 --- a/yarn-project/prover-node/src/job/epoch-proving-job.test.ts +++ b/yarn-project/prover-node/src/job/epoch-proving-job.test.ts @@ -48,7 +48,7 @@ describe('epoch-proving-job', () => { const NUM_TXS = NUM_BLOCKS * TXS_PER_BLOCK; // Subject factory - const createJob = (opts: { deadline?: Date; parallelBlockLimit?: number } = {}) => { + const createJob = (opts: { deadline?: Date; parallelBlockLimit?: number; skipSubmitProof?: boolean } = {}) => { const txsMap = new Map(txs.map(tx => [tx.getTxHash().toString(), tx])); const data: EpochProvingJobData = { @@ -68,7 +68,7 @@ describe('epoch-proving-job', () => { l2BlockSource, metrics, opts.deadline, - { parallelBlockLimit: opts.parallelBlockLimit ?? 32 }, + { parallelBlockLimit: opts.parallelBlockLimit ?? 32, skipSubmitProof: opts.skipSubmitProof }, ); }; @@ -201,4 +201,13 @@ describe('epoch-proving-job', () => { expect(publisher.submitEpochProof).not.toHaveBeenCalled(); expect(prover.cancel).toHaveBeenCalled(); }); + + it('skips publishing when skipSubmitProof is enabled', async () => { + const job = createJob({ skipSubmitProof: true }); + await job.run(); + + expect(job.getState()).toEqual('completed'); + expect(prover.finalizeEpoch).toHaveBeenCalled(); + expect(publisher.submitEpochProof).not.toHaveBeenCalled(); + }); }); diff --git a/yarn-project/prover-node/src/job/epoch-proving-job.ts b/yarn-project/prover-node/src/job/epoch-proving-job.ts index 9f0e23e07a89..05ade107af50 100644 --- a/yarn-project/prover-node/src/job/epoch-proving-job.ts +++ b/yarn-project/prover-node/src/job/epoch-proving-job.ts @@ -24,6 +24,12 @@ import type { ProverNodeJobMetrics } from '../metrics.js'; import type { ProverNodePublisher } from '../prover-node-publisher.js'; import { type EpochProvingJobData, validateEpochProvingJobData } from './epoch-proving-job-data.js'; +export type EpochProvingJobOptions = { + parallelBlockLimit?: number; + skipEpochCheck?: boolean; + skipSubmitProof?: boolean; +}; + /** * Job that grabs a range of blocks from the unfinalized chain from L1, gets their txs given their hashes, * re-executes their public calls, generates a rollup proof, and submits it to L1. This job will update the @@ -49,7 +55,7 @@ export class EpochProvingJob implements Traceable { private l2BlockSource: L2BlockSource | undefined, private metrics: ProverNodeJobMetrics, private deadline: Date | undefined, - private config: { parallelBlockLimit?: number; skipEpochCheck?: boolean }, + private config: EpochProvingJobOptions, ) { validateEpochProvingJobData(data); this.uuid = crypto.randomUUID(); @@ -178,6 +184,15 @@ export class EpochProvingJob implements Traceable { this.progressState('publishing-proof'); + if (this.config.skipSubmitProof) { + this.log.info( + `Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (blocks ${fromBlock} to ${toBlock})`, + ); + this.state = 'completed'; + this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs); + return; + } + const success = await this.publisher.submitEpochProof({ fromBlock, toBlock, diff --git a/yarn-project/prover-node/src/prover-node-publisher.ts b/yarn-project/prover-node/src/prover-node-publisher.ts index d1fc0b2e6dd7..7afdcf27868e 100644 --- a/yarn-project/prover-node/src/prover-node-publisher.ts +++ b/yarn-project/prover-node/src/prover-node-publisher.ts @@ -35,7 +35,6 @@ export type L1SubmitEpochProofArgs = { }; export class ProverNodePublisher { - private enabled: boolean; private interruptibleSleep = new InterruptibleSleep(); private sleepTimeMs: number; private interrupted = false; @@ -55,7 +54,6 @@ export class ProverNodePublisher { telemetry?: TelemetryClient; }, ) { - this.enabled = config.publisherEnabled ?? true; this.sleepTimeMs = config?.l1PublishRetryIntervalMS ?? 60_000; const telemetry = deps.telemetry ?? getTelemetryClient(); @@ -101,12 +99,6 @@ export class ProverNodePublisher { }): Promise { const { epochNumber, fromBlock, toBlock } = args; const ctx = { epochNumber, fromBlock, toBlock }; - - if (!this.enabled) { - this.log.warn(`Publishing L1 txs is disabled`); - return false; - } - if (!this.interrupted) { const timer = new Timer(); // Validate epoch proof range and hashes are correct before submitting diff --git a/yarn-project/prover-node/src/prover-node.test.ts b/yarn-project/prover-node/src/prover-node.test.ts index a445fa590c4d..52dd7c957b8c 100644 --- a/yarn-project/prover-node/src/prover-node.test.ts +++ b/yarn-project/prover-node/src/prover-node.test.ts @@ -108,6 +108,7 @@ describe('prover-node', () => { txGatheringMaxParallelRequestsPerNode: 5, proverNodeFailedEpochStore: undefined, txGatheringTimeoutMs: 1000, + proverNodeDisableProofPublish: false, }; // World state returns a new mock db every time it is asked to fork diff --git a/yarn-project/prover-node/src/prover-node.ts b/yarn-project/prover-node/src/prover-node.ts index 610a7237716e..dd47d0716718 100644 --- a/yarn-project/prover-node/src/prover-node.ts +++ b/yarn-project/prover-node/src/prover-node.ts @@ -366,7 +366,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable publisher: ProverNodePublisher, opts: { skipEpochCheck?: boolean } = {}, ) { - const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit } = this.config; + const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config; return new EpochProvingJob( data, this.worldState, @@ -376,7 +376,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable this.l2BlockSource, this.jobMetrics, deadline, - { parallelBlockLimit, ...opts }, + { parallelBlockLimit, skipSubmitProof: proverNodeDisableProofPublish, ...opts }, ); } diff --git a/yarn-project/sequencer-client/src/publisher/config.ts b/yarn-project/sequencer-client/src/publisher/config.ts index d8edf16735d8..b0526cf12493 100644 --- a/yarn-project/sequencer-client/src/publisher/config.ts +++ b/yarn-project/sequencer-client/src/publisher/config.ts @@ -26,9 +26,6 @@ export type TxSenderConfig = L1ReaderConfig & { * Publisher addresses to be used with a remote signer */ publisherAddresses?: EthAddress[]; - - /** Whether this publisher is enabled */ - publisherEnabled?: boolean; }; /** @@ -59,11 +56,6 @@ export const getTxSenderConfigMappings: ( parseEnv: (val: string) => val.split(',').map(address => EthAddress.fromString(address)), defaultValue: [], }, - publisherEnabled: { - env: scope === 'PROVER' ? `PROVER_PUBLISHER_ENABLED` : `SEQ_PUBLISHER_ENABLED`, - description: 'Whether this L1 publisher is enabled', - ...booleanConfigHelper(true), - }, }); export function getTxSenderConfigFromEnv(scope: 'PROVER' | 'SEQ'): Omit { diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts index cfd50ef08d6f..0e7f881b59c9 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts @@ -101,7 +101,6 @@ interface RequestWithExpiry { } export class SequencerPublisher { - private enabled: boolean; private interrupted = false; private metrics: SequencerPublisherMetrics; public epochCache: EpochCache; @@ -151,7 +150,6 @@ export class SequencerPublisher { log?: Logger; }, ) { - this.enabled = config.publisherEnabled ?? true; this.log = deps.log ?? createLogger('sequencer:publisher'); this.ethereumSlotDuration = BigInt(config.ethereumSlotDuration); this.epochCache = deps.epochCache; @@ -201,14 +199,6 @@ export class SequencerPublisher { * - undefined if no valid requests are found OR the tx failed to send. */ public async sendRequests() { - if (!this.enabled) { - this.log.warn(`Sending L1 txs is disabled`, { - requestsDiscarded: this.requests.map(r => r.action), - }); - this.requests = []; - return undefined; - } - const requestsToProcess = [...this.requests]; this.requests = []; if (this.interrupted) {