diff --git a/yarn-project/aztec/src/cli/aztec_start_options.ts b/yarn-project/aztec/src/cli/aztec_start_options.ts index 94500aa0d9eb..64d93b414c95 100644 --- a/yarn-project/aztec/src/cli/aztec_start_options.ts +++ b/yarn-project/aztec/src/cli/aztec_start_options.ts @@ -106,7 +106,7 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = { configToFlag('--auto-update-url', sharedNodeConfigMappings.autoUpdateUrl), configToFlag('--sync-mode', sharedNodeConfigMappings.syncMode), - configToFlag('--snapshots-url', sharedNodeConfigMappings.snapshotsUrl), + configToFlag('--snapshots-urls', sharedNodeConfigMappings.snapshotsUrls), ], SANDBOX: [ { diff --git a/yarn-project/cli/src/config/chain_l2_config.ts b/yarn-project/cli/src/config/chain_l2_config.ts index 0bbe9f1efff7..1f96b600332b 100644 --- a/yarn-project/cli/src/config/chain_l2_config.ts +++ b/yarn-project/cli/src/config/chain_l2_config.ts @@ -28,7 +28,7 @@ export type L2ChainConfig = L1ContractsConfig & seqMinTxsPerBlock: number; seqMaxTxsPerBlock: number; realProofs: boolean; - snapshotsUrl: string; + snapshotsUrls: string[]; autoUpdate: SharedNodeConfig['autoUpdate']; autoUpdateUrl?: string; maxTxPoolSize: number; @@ -105,7 +105,7 @@ export const stagingIgnitionL2ChainConfig: L2ChainConfig = { seqMinTxsPerBlock: 0, seqMaxTxsPerBlock: 0, realProofs: true, - snapshotsUrl: `${SNAPSHOT_URL}/staging-ignition/`, + snapshotsUrls: [`${SNAPSHOT_URL}/staging-ignition/`], autoUpdate: 'config-and-version', autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/staging-ignition.json', maxTxPoolSize: 100_000_000, // 100MB @@ -188,7 +188,7 @@ export const stagingPublicL2ChainConfig: L2ChainConfig = { seqMinTxsPerBlock: 0, seqMaxTxsPerBlock: 20, realProofs: true, - snapshotsUrl: `${SNAPSHOT_URL}/staging-public/`, + snapshotsUrls: [`${SNAPSHOT_URL}/staging-public/`], autoUpdate: 'config-and-version', autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/staging-public.json', publicIncludeMetrics, @@ -243,7 +243,7 @@ export const testnetL2ChainConfig: L2ChainConfig = { seqMinTxsPerBlock: 0, seqMaxTxsPerBlock: 20, realProofs: true, - snapshotsUrl: `${SNAPSHOT_URL}/testnet/`, + snapshotsUrls: [`${SNAPSHOT_URL}/testnet/`], autoUpdate: 'config-and-version', autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/testnet.json', maxTxPoolSize: 100_000_000, // 100MB @@ -300,7 +300,7 @@ export const ignitionL2ChainConfig: L2ChainConfig = { seqMinTxsPerBlock: 0, seqMaxTxsPerBlock: 0, realProofs: true, - snapshotsUrl: 'https://storage.googleapis.com/aztec-testnet/snapshots/ignition/', + snapshotsUrls: ['https://storage.googleapis.com/aztec-testnet/snapshots/ignition/'], autoUpdate: 'notify', autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/ignition.json', maxTxPoolSize: 100_000_000, // 100MB @@ -436,7 +436,7 @@ export async function enrichEnvironmentWithChainConfig(networkName: NetworkNames enrichVar('SEQ_MAX_TX_PER_BLOCK', config.seqMaxTxsPerBlock.toString()); enrichVar('PROVER_REAL_PROOFS', config.realProofs.toString()); enrichVar('PXE_PROVER_ENABLED', config.realProofs.toString()); - enrichVar('SYNC_SNAPSHOTS_URL', config.snapshotsUrl); + enrichVar('SYNC_SNAPSHOTS_URLS', config.snapshotsUrls.join(',')); enrichVar('P2P_MAX_TX_POOL_SIZE', config.maxTxPoolSize.toString()); enrichVar('DATA_STORE_MAP_SIZE_KB', config.dbMapSizeKb.toString()); diff --git a/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts b/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts index eb82a8365d0e..07b9395ca0cb 100644 --- a/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts +++ b/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts @@ -4,11 +4,11 @@ import { RollupContract } from '@aztec/ethereum'; import { ChainMonitor } from '@aztec/ethereum/test'; import { randomBytes } from '@aztec/foundation/crypto'; import { tryRmDir } from '@aztec/foundation/fs'; -import { withLogNameSuffix } from '@aztec/foundation/log'; +import { logger, withLogNameSuffix } from '@aztec/foundation/log'; import { bufferToHex } from '@aztec/foundation/string'; import { ProverNode, type ProverNodeConfig } from '@aztec/prover-node'; -import { mkdtemp, readdir } from 'fs/promises'; +import { cp, mkdtemp, readFile, readdir, rm, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -24,6 +24,8 @@ describe('e2e_snapshot_sync', () => { let snapshotDir: string; let snapshotLocation: string; + let cleanupDirs: string[]; + beforeAll(async () => { context = await setup(0, { minTxsPerBlock: 0, @@ -38,6 +40,7 @@ describe('e2e_snapshot_sync', () => { log = context.logger; snapshotDir = await mkdtemp(join(tmpdir(), 'snapshots-')); + cleanupDirs = [snapshotDir]; snapshotLocation = `file://${snapshotDir}`; monitor = new ChainMonitor(RollupContract.getFromConfig(context.config), context.dateProvider, log).start(); }); @@ -45,7 +48,7 @@ describe('e2e_snapshot_sync', () => { afterAll(async () => { await monitor.stop(); await context.teardown(); - await tryRmDir(snapshotDir, log); + await Promise.all(cleanupDirs.map(dir => tryRmDir(dir, log))); }); // Adapted from epochs-test @@ -95,7 +98,7 @@ describe('e2e_snapshot_sync', () => { it('downloads snapshot when syncing new node', async () => { log.warn(`Syncing brand new node with snapshot sync`); - const node = await createNonValidatorNode('1', { snapshotsUrl: snapshotLocation, syncMode: 'snapshot' }); + const node = await createNonValidatorNode('1', { snapshotsUrls: [snapshotLocation], syncMode: 'snapshot' }); log.warn(`New node synced`); await expectNodeSyncedToL2Block(node, L2_TARGET_BLOCK_NUM); @@ -116,7 +119,7 @@ describe('e2e_snapshot_sync', () => { it('downloads snapshot when syncing new prover node', async () => { log.warn(`Syncing brand new prover node with snapshot sync`); - const node = await createTestProverNode({ snapshotsUrl: snapshotLocation, syncMode: 'snapshot' }); + const node = await createTestProverNode({ snapshotsUrls: [snapshotLocation], syncMode: 'snapshot' }); log.warn(`New node prover synced`); await expectNodeSyncedToL2Block(node, L2_TARGET_BLOCK_NUM); @@ -124,4 +127,108 @@ describe('e2e_snapshot_sync', () => { log.warn(`Stopping new prover node`); await node.stop(); }); + + it('downloads snapshot from multiple sources', async () => { + log.warn(`Setting up multiple snapshot locations with different L1 block heights`); + + // Create two additional snapshot directories (third one is the existing snapshotDir) + const snapshotDir1 = await mkdtemp(join(tmpdir(), 'snapshots-1-')); + const snapshotDir2 = await mkdtemp(join(tmpdir(), 'snapshots-2-')); + const snapshotLocation1 = `file://${snapshotDir1}`; + const snapshotLocation2 = `file://${snapshotDir2}`; + const snapshotLocation3 = snapshotLocation; // Use the existing snapshot + + cleanupDirs.push(snapshotDir1, snapshotDir2); + + // Copy the existing snapshot to snapshot 1 and 2 + log.warn(`Copying existing snapshot to two new locations`); + const originalFiles = await readdir(snapshotDir, { recursive: true }); + log.warn(`Found ${originalFiles.length} files in snapshot directory`); + + // Find the index.json file + const indexFile = originalFiles.find(f => typeof f === 'string' && f.includes('index.json')); + expect(indexFile).toBeDefined(); + + // Copy all files recursively + for (const file of originalFiles) { + const srcPath = join(snapshotDir, file as string); + const destPath1 = join(snapshotDir1, file as string); + const destPath2 = join(snapshotDir2, file as string); + + try { + await cp(srcPath, destPath1, { recursive: true }); + await cp(srcPath, destPath2, { recursive: true }); + } catch { + // Skip if it's a directory or already copied + } + } + + // Update index jsons + for (const newDir of [snapshotDir1, snapshotDir2]) { + const files = await readdir(newDir, { recursive: true }); + const indexFile = files.find(f => typeof f === 'string' && f.includes('index.json')); + expect(indexFile).toBeDefined(); + const indexContents = await readFile(join(newDir, indexFile!), 'utf-8'); + const updatedContents = indexContents.replaceAll(snapshotDir, newDir); + await writeFile(join(newDir, indexFile!), updatedContents); + logger.info(`Updated index file in ${newDir}`, { updatedContents }); + } + + // Read the original index.json to get the base L1 block number + const indexPath3 = join(snapshotDir, indexFile!); + const indexContent = JSON.parse(await readFile(indexPath3, 'utf-8')); + const baseL1Block = indexContent.snapshots[0].l1BlockNumber; + log.warn(`Base L1 block number: ${baseL1Block}`); + + // Modify snapshot 1: increase L1 block height (highest) and corrupt it + log.warn(`Modifying snapshot 1 to have highest L1 block height`); + const indexPath1 = join(snapshotDir1, indexFile!); + const index1 = JSON.parse(await readFile(indexPath1, 'utf-8')); + index1.snapshots[0].l1BlockNumber = baseL1Block + 200; // Highest + await writeFile(indexPath1, JSON.stringify(index1, null, 2)); + + // Corrupt snapshot 1 by removing one of the database files + log.warn(`Corrupting snapshot 1 by removing a database file`); + const snapshot1Files = await readdir(snapshotDir1, { recursive: true }); + const dbFile = snapshot1Files.find(f => typeof f === 'string' && f.endsWith('.db')); + expect(dbFile).toBeDefined(); + await rm(join(snapshotDir1, dbFile!)); + log.warn(`Removed ${dbFile} from snapshot 1`); + + // Modify snapshot 2: decrease L1 block height (lowest) + log.warn(`Modifying snapshot 2 to have lowest L1 block height`); + const indexPath2 = join(snapshotDir2, indexFile!); + const index2 = JSON.parse(await readFile(indexPath2, 'utf-8')); + index2.snapshots[0].l1BlockNumber = baseL1Block - 1; // Lowest + await writeFile(indexPath2, JSON.stringify(index2, null, 2)); + + // Snapshot 3 (original) has the middle L1 block height (baseL1Block) + log.warn(`Snapshot 3 (original) has L1 block height ${baseL1Block} (middle)`); + + // Now sync a new node with all three URLs + // Snapshot 1: highest L1 block (baseL1Block + 200) but corrupted (should fail) + // Snapshot 2: lowest L1 block (baseL1Block - 1) but valid + // Snapshot 3: middle L1 block (baseL1Block) and valid (should be selected after 1 fails) + log.warn(`Syncing brand new node with three snapshot URLs`); + const node = await createNonValidatorNode('multi-url', { + snapshotsUrls: [snapshotLocation1, snapshotLocation2, snapshotLocation3], + syncMode: 'snapshot', + }); + + log.warn(`New node synced with fallback logic`); + await expectNodeSyncedToL2Block(node, L2_TARGET_BLOCK_NUM); + + const block = await node.getBlock(L2_TARGET_BLOCK_NUM); + expect(block).toBeDefined(); + const blockHash = await block!.hash(); + + log.warn(`Checking for L2 block ${L2_TARGET_BLOCK_NUM} with hash ${blockHash} on both nodes`); + const getBlockHashLeafIndex = (node: AztecNode) => + node.findLeavesIndexes(L2_TARGET_BLOCK_NUM, MerkleTreeId.ARCHIVE, [blockHash]).then(([i]) => i); + expect(await getBlockHashLeafIndex(context.aztecNode)).toBeDefined(); + expect(await getBlockHashLeafIndex(node)).toBeDefined(); + + log.warn(`Stopping new node`); + await node.stop(); + }); }); diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index dd8c7210a63c..9f264c49034b 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -212,6 +212,7 @@ export type EnvVar = | 'SLASH_MAX_PAYLOAD_SIZE' | 'SLASH_EXECUTE_ROUNDS_LOOK_BACK' | 'SYNC_MODE' + | 'SYNC_SNAPSHOTS_URLS' | 'SYNC_SNAPSHOTS_URL' | 'TELEMETRY' | 'TEST_ACCOUNTS' diff --git a/yarn-project/node-lib/src/actions/snapshot-sync.ts b/yarn-project/node-lib/src/actions/snapshot-sync.ts index ba2d61c4c834..6c81fb18c52e 100644 --- a/yarn-project/node-lib/src/actions/snapshot-sync.ts +++ b/yarn-project/node-lib/src/actions/snapshot-sync.ts @@ -26,11 +26,12 @@ import type { SharedNodeConfig } from '../config/index.js'; // Half day worth of L1 blocks const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12; -type SnapshotSyncConfig = Pick & +type SnapshotSyncConfig = Pick & Pick & Pick & Required & EthereumClientConfig & { + snapshotsUrls?: string[]; minL1BlocksToTriggerReplace?: number; }; @@ -39,14 +40,14 @@ type SnapshotSyncConfig = Pick & * Behaviour depends on syncing mode. */ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) { - const { syncMode, snapshotsUrl, dataDirectory, l1ChainId, rollupVersion, l1Contracts } = config; + const { syncMode, snapshotsUrls, dataDirectory, l1ChainId, rollupVersion, l1Contracts } = config; if (syncMode === 'full') { log.debug('Snapshot sync is disabled. Running full sync.', { syncMode: syncMode }); return false; } - if (!snapshotsUrl) { - log.verbose('Snapshot sync is disabled. No snapshots URL provided.'); + if (!snapshotsUrls || snapshotsUrls.length === 0) { + log.verbose('Snapshot sync is disabled. No snapshots URLs provided.'); return false; } @@ -55,15 +56,7 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) { return false; } - let fileStore: ReadOnlyFileStore; - try { - fileStore = await createReadOnlyFileStore(snapshotsUrl, log); - } catch (err) { - log.error(`Invalid config for downloading snapshots`, err); - return false; - } - - // Create an archiver store to check the current state + // Create an archiver store to check the current state (do this only once) log.verbose(`Creating temporary archiver data store`); const archiverStore = await createArchiverStore(config); let archiverL1BlockNumber: bigint | undefined; @@ -102,65 +95,111 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) { rollupVersion, rollupAddress: l1Contracts.rollupAddress, }; - let snapshot: SnapshotMetadata | undefined; - try { - snapshot = await getLatestSnapshotMetadata(indexMetadata, fileStore); - } catch (err) { - log.error(`Failed to get latest snapshot metadata. Skipping snapshot sync.`, err, { - ...indexMetadata, - snapshotsUrl, - }); - return false; - } - if (!snapshot) { - log.verbose(`No snapshot found. Skipping snapshot sync.`, { ...indexMetadata, snapshotsUrl }); - return false; - } + // Fetch latest snapshot from each URL + type SnapshotCandidate = { snapshot: SnapshotMetadata; url: string; fileStore: ReadOnlyFileStore }; + const snapshotCandidates: SnapshotCandidate[] = []; - if (snapshot.schemaVersions.archiver !== ARCHIVER_DB_VERSION) { - log.warn( - `Skipping snapshot sync as last snapshot has schema version ${snapshot.schemaVersions.archiver} but expected ${ARCHIVER_DB_VERSION}.`, - snapshot, - ); - return false; - } + for (const snapshotsUrl of snapshotsUrls) { + let fileStore: ReadOnlyFileStore; + try { + fileStore = await createReadOnlyFileStore(snapshotsUrl, log); + } catch (err) { + log.error(`Invalid config for downloading snapshots from ${snapshotsUrl}`, err); + continue; + } - if (snapshot.schemaVersions.worldState !== WORLD_STATE_DB_VERSION) { - log.warn( - `Skipping snapshot sync as last snapshot has world state schema version ${snapshot.schemaVersions.worldState} but we expected ${WORLD_STATE_DB_VERSION}.`, - snapshot, - ); - return false; + let snapshot: SnapshotMetadata | undefined; + try { + snapshot = await getLatestSnapshotMetadata(indexMetadata, fileStore); + } catch (err) { + log.error(`Failed to get latest snapshot metadata from ${snapshotsUrl}. Skipping this URL.`, err, { + ...indexMetadata, + snapshotsUrl, + }); + continue; + } + + if (!snapshot) { + log.verbose(`No snapshot found at ${snapshotsUrl}. Skipping this URL.`, { ...indexMetadata, snapshotsUrl }); + continue; + } + + if (snapshot.schemaVersions.archiver !== ARCHIVER_DB_VERSION) { + log.warn( + `Skipping snapshot from ${snapshotsUrl} as it has schema version ${snapshot.schemaVersions.archiver} but expected ${ARCHIVER_DB_VERSION}.`, + snapshot, + ); + continue; + } + + if (snapshot.schemaVersions.worldState !== WORLD_STATE_DB_VERSION) { + log.warn( + `Skipping snapshot from ${snapshotsUrl} as it has world state schema version ${snapshot.schemaVersions.worldState} but we expected ${WORLD_STATE_DB_VERSION}.`, + snapshot, + ); + continue; + } + + if (archiverL1BlockNumber && snapshot.l1BlockNumber < archiverL1BlockNumber) { + log.verbose( + `Skipping snapshot from ${snapshotsUrl} since local archiver is at L1 block ${archiverL1BlockNumber} which is further than snapshot at ${snapshot.l1BlockNumber}`, + { snapshot, archiverL1BlockNumber, snapshotsUrl }, + ); + continue; + } + + if (archiverL1BlockNumber && snapshot.l1BlockNumber - Number(archiverL1BlockNumber) < minL1BlocksToTriggerReplace) { + log.verbose( + `Skipping snapshot from ${snapshotsUrl} as archiver is less than ${ + snapshot.l1BlockNumber - Number(archiverL1BlockNumber) + } L1 blocks behind this snapshot.`, + { snapshot, archiverL1BlockNumber, snapshotsUrl }, + ); + continue; + } + + snapshotCandidates.push({ snapshot, url: snapshotsUrl, fileStore }); } - if (archiverL1BlockNumber && snapshot.l1BlockNumber < archiverL1BlockNumber) { - log.verbose( - `Skipping snapshot sync since local archiver is at L1 block ${archiverL1BlockNumber} which is further than last snapshot at ${snapshot.l1BlockNumber}`, - { snapshot, archiverL1BlockNumber }, - ); + if (snapshotCandidates.length === 0) { + log.verbose(`No valid snapshots found from any URL. Skipping snapshot sync.`, { ...indexMetadata, snapshotsUrls }); return false; } - if (archiverL1BlockNumber && snapshot.l1BlockNumber - Number(archiverL1BlockNumber) < minL1BlocksToTriggerReplace) { - log.verbose( - `Skipping snapshot sync as archiver is less than ${ - snapshot.l1BlockNumber - Number(archiverL1BlockNumber) - } L1 blocks behind latest snapshot.`, - { snapshot, archiverL1BlockNumber }, - ); - return false; + // Sort candidates by L1 block number (highest first) + snapshotCandidates.sort((a, b) => b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber); + + // Try each candidate in order until one succeeds + for (const { snapshot, url } of snapshotCandidates) { + const { l1BlockNumber, l2BlockNumber } = snapshot; + log.info(`Attempting to sync from snapshot at L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, { + snapshot, + snapshotsUrl: url, + }); + + try { + await snapshotSync(snapshot, log, { + dataDirectory: config.dataDirectory!, + rollupAddress: config.l1Contracts.rollupAddress, + snapshotsUrl: url, + }); + log.info(`Snapshot synced to L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, { + snapshot, + snapshotsUrl: url, + }); + return true; + } catch (err) { + log.error(`Failed to download snapshot from ${url}. Trying next candidate.`, err, { + snapshot, + snapshotsUrl: url, + }); + continue; + } } - const { l1BlockNumber, l2BlockNumber } = snapshot; - log.info(`Syncing from snapshot at L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, { snapshot, snapshotsUrl }); - await snapshotSync(snapshot, log, { - dataDirectory: config.dataDirectory!, - rollupAddress: config.l1Contracts.rollupAddress, - snapshotsUrl, - }); - log.info(`Snapshot synced to L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, { snapshot }); - return true; + log.error(`Failed to download snapshot from all URLs.`, { snapshotsUrls }); + return false; } /** diff --git a/yarn-project/node-lib/src/config/index.ts b/yarn-project/node-lib/src/config/index.ts index e3ce5d56005f..f59f54ab7f05 100644 --- a/yarn-project/node-lib/src/config/index.ts +++ b/yarn-project/node-lib/src/config/index.ts @@ -7,8 +7,8 @@ export type SharedNodeConfig = { sponsoredFPC: boolean; /** Sync mode: full to always sync via L1, snapshot to download a snapshot if there is no local data, force-snapshot to download even if there is local data. */ syncMode: 'full' | 'snapshot' | 'force-snapshot'; - /** Base URL for snapshots index. Index file will be searched at `SNAPSHOTS_BASE_URL/aztec-L1_CHAIN_ID-VERSION-ROLLUP_ADDRESS/index.json` */ - snapshotsUrl?: string; + /** Base URLs for snapshots index. Index file will be searched at `SNAPSHOTS_BASE_URL/aztec-L1_CHAIN_ID-VERSION-ROLLUP_ADDRESS/index.json` */ + snapshotsUrls?: string[]; /** Auto update mode: disabled - to completely ignore remote signals to update the node. enabled - to respect the signals (potentially shutting this node down). log - check for updates but log a warning instead of applying them*/ autoUpdate?: 'disabled' | 'notify' | 'config' | 'config-and-version'; @@ -36,9 +36,16 @@ export const sharedNodeConfigMappings: ConfigMappingsType = { 'Set sync mode to `full` to always sync via L1, `snapshot` to download a snapshot if there is no local data, `force-snapshot` to download even if there is local data.', defaultValue: 'snapshot', }, - snapshotsUrl: { - env: 'SYNC_SNAPSHOTS_URL', - description: 'Base URL for snapshots index.', + snapshotsUrls: { + env: 'SYNC_SNAPSHOTS_URLS', + description: 'Base URLs for snapshots index, comma-separated.', + parseEnv: (val: string) => + val + .split(',') + .map(url => url.trim()) + .filter(url => url.length > 0), + fallback: ['SYNC_SNAPSHOTS_URL'], + defaultValue: [], }, autoUpdate: { env: 'AUTO_UPDATE',