feat: Prover node snapshot sync

yarn-project/aztec-node/package.json

@@ -72,6 +72,7 @@
     "@aztec/foundation": "workspace:^",
     "@aztec/kv-store": "workspace:^",
     "@aztec/merkle-tree": "workspace:^",
+    "@aztec/node-lib": "workspace:^",
     "@aztec/p2p": "workspace:^",
     "@aztec/protocol-contracts": "workspace:^",
     "@aztec/prover-client": "workspace:^",

yarn-project/aztec-node/src/aztec-node/config.ts

@@ -2,6 +2,7 @@ import { type ArchiverConfig, archiverConfigMappings } from '@aztec/archiver/con
 import { type L1ContractAddresses, l1ContractAddressesMapping } from '@aztec/ethereum';
 import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings } from '@aztec/foundation/config';
 import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config';
+import { type SharedNodeConfig, sharedNodeConfigMappings } from '@aztec/node-lib/config';
 import { type P2PConfig, p2pConfigMappings } from '@aztec/p2p/config';
 import { type ProverClientConfig, proverClientConfigMappings } from '@aztec/prover-client/config';
 import { type SequencerClientConfig, sequencerClientConfigMappings } from '@aztec/sequencer-client/config';
@@ -27,19 +28,12 @@ export type AztecNodeConfig = ArchiverConfig &
   Pick<ProverClientConfig, 'bbBinaryPath' | 'bbWorkingDirectory' | 'realProofs'> &
   P2PConfig &
   DataStoreConfig &
-  SentinelConfig & {
-    /** Whether the validator is disabled for this node */
-    disableValidator: boolean;
-    /** Whether to populate the genesis state with initial fee juice for the test accounts */
-    testAccounts: boolean;
-    /** Whether to populate the genesis state with initial fee juice for the sponsored FPC */
-    sponsoredFPC: boolean;
+  SentinelConfig &
+  SharedNodeConfig & {
     /** L1 contracts addresses */
     l1Contracts: L1ContractAddresses;
-    /** Sync mode: full to always sync via L1, snapshot to download a snapshot if there is no local data, force-snapshot to download even if there is local data. */
-    syncMode: 'full' | 'snapshot' | 'force-snapshot';
-    /** Base URL for snapshots index. Index file will be searched at `SNAPSHOTS_BASE_URL/aztec-L1_CHAIN_ID-VERSION-ROLLUP_ADDRESS/index.json` */
-    snapshotsUrl?: string;
+    /** Whether the validator is disabled for this node */
+    disableValidator: boolean;
   };
 
 export const aztecNodeConfigMappings: ConfigMappingsType<AztecNodeConfig> = {
@@ -51,6 +45,7 @@ export const aztecNodeConfigMappings: ConfigMappingsType<AztecNodeConfig> = {
   ...worldStateConfigMappings,
   ...p2pConfigMappings,
   ...sentinelConfigMappings,
+  ...sharedNodeConfigMappings,
   l1Contracts: {
     description: 'The deployed L1 contract addresses',
     nested: l1ContractAddressesMapping,
@@ -60,26 +55,6 @@ export const aztecNodeConfigMappings: ConfigMappingsType<AztecNodeConfig> = {
     description: 'Whether the validator is disabled for this node.',
     ...booleanConfigHelper(),
   },
-  testAccounts: {
-    env: 'TEST_ACCOUNTS',
-    description: 'Whether to populate the genesis state with initial fee juice for the test accounts.',
-    ...booleanConfigHelper(),
-  },
-  sponsoredFPC: {
-    env: 'SPONSORED_FPC',
-    description: 'Whether to populate the genesis state with initial fee juice for the sponsored FPC.',
-    ...booleanConfigHelper(false),
-  },
-  syncMode: {
-    env: 'SYNC_MODE',
-    description:
-      'Set sync mode to `full` to always sync via L1, `snapshot` to download a snapshot if there is no local data, `force-snapshot` to download even if there is local data.',
-    defaultValue: 'snapshot',
-  },
-  snapshotsUrl: {
-    env: 'SYNC_SNAPSHOTS_URL',
-    description: 'Base URL for snapshots index.',
-  },
 };
 
 /**

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -21,6 +21,7 @@ import { SiblingPath } from '@aztec/foundation/trees';
 import type { AztecKVStore } from '@aztec/kv-store';
 import { openTmpStore } from '@aztec/kv-store/lmdb';
 import { SHA256Trunc, StandardTree, UnbalancedTree } from '@aztec/merkle-tree';
+import { trySnapshotSync, uploadSnapshot } from '@aztec/node-lib/actions';
 import { type P2P, createP2PClient } from '@aztec/p2p';
 import { ProtocolContractAddress } from '@aztec/protocol-contracts';
 import {
@@ -86,8 +87,6 @@ import {
 import { createValidatorClient } from '@aztec/validator-client';
 import { createWorldStateSynchronizer } from '@aztec/world-state';
 
-import { trySnapshotSync } from '../actions/snapshot-sync.js';
-import { uploadSnapshot } from '../actions/upload-snapshot.js';
 import { createSentinel } from '../sentinel/factory.js';
 import { Sentinel } from '../sentinel/sentinel.js';
 import { type AztecNodeConfig, getPackageVersion } from './config.js';
@@ -174,7 +173,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       );
     }
 
-    // attempt fast sync if needed
+    // attempt snapshot sync if possible
     await trySnapshotSync(config, log);
 
     const archiver = await createArchiver(config, blobSinkClient, { blockUntilSync: true }, telemetry);

yarn-project/aztec-node/tsconfig.json

@@ -33,6 +33,9 @@
     {
       "path": "../merkle-tree"
     },
+    {
+      "path": "../node-lib"
+    },
     {
       "path": "../p2p"
     },

yarn-project/aztec/src/cli/aztec_start_options.ts

@@ -282,7 +282,7 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = {
     },
     {
       flag: '--node.snapshotsUrl <value>',
-      description: 'Base URL for downloading snapshots for fast sync.',
+      description: 'Base URL for downloading snapshots for snapshot sync.',
       defaultValue: undefined,
       envVar: 'SYNC_SNAPSHOTS_URL',
     },

yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts

@@ -5,6 +5,7 @@ import { ChainMonitor } from '@aztec/ethereum/test';
 import { randomBytes } from '@aztec/foundation/crypto';
 import { tryRmDir } from '@aztec/foundation/fs';
 import { withLogNameSuffix } from '@aztec/foundation/log';
+import { type ProverNodeConfig, createProverNode } from '@aztec/prover-node';
 
 import { mkdtemp, readdir } from 'fs/promises';
 import { tmpdir } from 'os';
@@ -59,6 +60,28 @@ describe('e2e_snapshot_sync', () => {
     );
   };
 
+  // Adapted from utils/createAndSyncProverNode
+  const createTestProverNode = async (suffix: string, config: Partial<ProverNodeConfig> = {}) => {
+    log.warn('Creating and syncing a prover node...');
+    return await withLogNameSuffix(suffix, () =>
+      createProverNode({
+        ...context.config,
+        dataDirectory: join(context.config.dataDirectory!, randomBytes(8).toString('hex')),
+        p2pEnabled: true, // So we don't need prover coordination
+        proverCoordinationNodeUrl: undefined,
+        proverNodeMaxPendingJobs: 10,
+        proverNodeMaxParallelBlocksPerEpoch: 32,
+        proverNodePollingIntervalMs: 200,
+        txGatheringTimeoutMs: 60000,
+        txGatheringIntervalMs: 1000,
+        txGatheringMaxParallelRequests: 100,
+        proverAgentCount: 1,
+        realProofs: false,
+        ...config,
+      }),
+    );
+  };
+
   it('waits until a few L2 blocks have been mined', async () => {
     log.warn(`Waiting for L2 blocks to be mined`);
     await retryUntil(() => monitor.l2BlockNumber > L2_TARGET_BLOCK_NUM, 'l2-blocks-mined', 90, 1);
@@ -72,7 +95,7 @@ describe('e2e_snapshot_sync', () => {
     log.warn(`Snapshot created`);
   });
 
-  it('downloads snapshot from new node', async () => {
+  it('downloads snapshot when syncing new node', async () => {
     log.warn(`Syncing brand new node with snapshot sync`);
     const node = await createNonValidatorNode('1', {
       blobSinkUrl: undefined, // set no blob sink so it cannot sync on its own
@@ -94,4 +117,23 @@ describe('e2e_snapshot_sync', () => {
     log.warn(`Stopping new node`);
     await node.stop();
   });
+
+  it('downloads snapshot when syncing new prover node', async () => {
+    log.warn(`Syncing brand new prover node with snapshot sync`);
+    const node = await createTestProverNode('1', {
+      blobSinkUrl: undefined, // set no blob sink so it cannot sync on its own
+      snapshotsUrl: snapshotLocation,
+      syncMode: 'snapshot',
+    });
+
+    log.warn(`New node prover synced`);
+    const tips = await node.getL2Tips();
+    expect(tips.latest.number).toBeLessThanOrEqual(L2_TARGET_BLOCK_NUM);
+
+    const worldState = await node.getWorldStateSyncStatus();
+    expect(worldState.latestBlockNumber).toBeLessThanOrEqual(L2_TARGET_BLOCK_NUM);
+
+    log.warn(`Stopping new prover node`);
+    await node.stop();
+  });
 });

yarn-project/node-lib/.eslintrc.cjs

@@ -0,0 +1 @@
+module.exports = require('@aztec/foundation/eslint');

yarn-project/node-lib/README.md

@@ -0,0 +1,3 @@
+# Node Lib
+
+Shared code for Aztec Nodes and Prover Nodes.

yarn-project/node-lib/package.json

@@ -0,0 +1,93 @@
+{
+  "name": "@aztec/node-lib",
+  "version": "0.1.0",
+  "type": "module",
+  "exports": {
+    "./actions": "./dest/actions/index.js",
+    "./config": "./dest/config/index.js"
+  },
+  "inherits": [
+    "../package.common.json"
+  ],
+  "scripts": {
+    "build": "yarn clean && tsc -b",
+    "build:dev": "tsc -b --watch",
+    "clean": "rm -rf ./dest .tsbuildinfo",
+    "formatting": "run -T prettier --check ./src && run -T eslint ./src",
+    "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
+    "bb": "node --no-warnings ./dest/bb/index.js",
+    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
+  },
+  "jest": {
+    "moduleNameMapper": {
+      "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+    },
+    "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
+    "rootDir": "./src",
+    "transform": {
+      "^.+\\.tsx?$": [
+        "@swc/jest",
+        {
+          "jsc": {
+            "parser": {
+              "syntax": "typescript",
+              "decorators": true
+            },
+            "transform": {
+              "decoratorVersion": "2022-03"
+            }
+          }
+        }
+      ]
+    },
+    "extensionsToTreatAsEsm": [
+      ".ts"
+    ],
+    "reporters": [
+      "default"
+    ],
+    "testTimeout": 120000,
+    "setupFiles": [
+      "../../foundation/src/jest/setup.mjs"
+    ]
+  },
+  "dependencies": {
+    "@aztec/archiver": "workspace:^",
+    "@aztec/bb-prover": "workspace:^",
+    "@aztec/blob-sink": "workspace:^",
+    "@aztec/constants": "workspace:^",
+    "@aztec/epoch-cache": "workspace:^",
+    "@aztec/ethereum": "workspace:^",
+    "@aztec/foundation": "workspace:^",
+    "@aztec/kv-store": "workspace:^",
+    "@aztec/merkle-tree": "workspace:^",
+    "@aztec/p2p": "workspace:^",
+    "@aztec/protocol-contracts": "workspace:^",
+    "@aztec/prover-client": "workspace:^",
+    "@aztec/sequencer-client": "workspace:^",
+    "@aztec/simulator": "workspace:^",
+    "@aztec/stdlib": "workspace:^",
+    "@aztec/telemetry-client": "workspace:^",
+    "@aztec/validator-client": "workspace:^",
+    "@aztec/world-state": "workspace:^",
+    "tslib": "^2.4.0"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.5.0",
+    "@types/jest": "^29.5.0",
+    "@types/node": "^18.7.23",
+    "jest": "^29.5.0",
+    "jest-mock-extended": "^3.0.3",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4"
+  },
+  "files": [
+    "dest",
+    "src",
+    "!*.test.*"
+  ],
+  "types": "./dest/index.d.ts",
+  "engines": {
+    "node": ">=18"
+  }
+}

yarn-project/node-lib/src/actions/index.ts

@@ -0,0 +1,2 @@
+export * from './snapshot-sync.js';
+export * from './upload-snapshot.js';

yarn-project/aztec-node/src/actions/snapshot-sync.ts → yarn-project/node-lib/src/actions/snapshot-sync.ts

@@ -6,11 +6,12 @@ import {
   createArchiverStore,
 } from '@aztec/archiver';
 import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
-import { type EthereumClientConfig, type L1ContractAddresses, getPublicClient } from '@aztec/ethereum';
+import { type EthereumClientConfig, getPublicClient } from '@aztec/ethereum';
 import type { EthAddress } from '@aztec/foundation/eth-address';
 import { tryRmDir } from '@aztec/foundation/fs';
 import type { Logger } from '@aztec/foundation/log';
 import type { DataStoreConfig } from '@aztec/kv-store/config';
+import type { ChainConfig } from '@aztec/stdlib/config';
 import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
 import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec/stdlib/file-store';
 import {
@@ -25,19 +26,16 @@ import { NATIVE_WORLD_STATE_DBS, WORLD_STATE_DB_VERSION, WORLD_STATE_DIR } from
 import { mkdir, mkdtemp, rename } from 'fs/promises';
 import { join } from 'path';
 
-import type { AztecNodeConfig } from '../aztec-node/config.js';
+import type { SharedNodeConfig } from '../config/index.js';
 
 // Half day worth of L1 blocks
 const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
 
-type SnapshotSyncConfig = Pick<
-  AztecNodeConfig,
-  'syncMode' | 'snapshotsUrl' | 'l1ChainId' | 'version' | 'dataDirectory'
-> &
+type SnapshotSyncConfig = Pick<SharedNodeConfig, 'syncMode' | 'snapshotsUrl'> &
+  Pick<ChainConfig, 'l1ChainId' | 'version'> &
   Pick<ArchiverConfig, 'archiverStoreMapSizeKb' | 'maxLogs'> &
-  DataStoreConfig &
+  Required<DataStoreConfig> &
   EthereumClientConfig & {
-    l1Contracts: Pick<L1ContractAddresses, 'rollupAddress'>;
     minL1BlocksToTriggerReplace?: number;
   };
 

yarn-project/aztec-node/src/actions/upload-snapshot.ts → yarn-project/node-lib/src/actions/upload-snapshot.ts

@@ -1,6 +1,8 @@
 import { ARCHIVER_DB_VERSION, type Archiver } from '@aztec/archiver';
 import { tryRmDir } from '@aztec/foundation/fs';
 import type { Logger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/kv-store/config';
+import type { ChainConfig } from '@aztec/stdlib/config';
 import { createFileStore } from '@aztec/stdlib/file-store';
 import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
 import type { SnapshotDataUrls, UploadSnapshotMetadata } from '@aztec/stdlib/snapshots';
@@ -12,7 +14,7 @@ import { mkdtemp } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join } from 'path';
 
-import type { AztecNodeConfig } from '../aztec-node/config.js';
+type UploadSnapshotConfig = Pick<ChainConfig, 'l1ChainId' | 'version'> & Pick<DataStoreConfig, 'dataDirectory'>;
 
 /**
  * Pauses the archiver and world state sync, creates backups of the archiver and world state lmdb environments,
@@ -22,7 +24,7 @@ export async function uploadSnapshot(
   location: string,
   archiver: Archiver,
   worldState: WorldStateSynchronizer,
-  config: Pick<AztecNodeConfig, 'dataDirectory' | 'l1ChainId' | 'version'>,
+  config: UploadSnapshotConfig,
   log: Logger,
 ) {
   const store = await createFileStore(location);
@@ -47,7 +49,7 @@ export async function uploadSnapshot(
 
 async function buildSnapshotMetadata(
   archiver: Archiver,
-  config: Pick<AztecNodeConfig, 'dataDirectory' | 'l1ChainId' | 'version'>,
+  config: UploadSnapshotConfig,
 ): Promise<UploadSnapshotMetadata> {
   const [rollupAddress, l1BlockNumber, { latest }] = await Promise.all([
     archiver.getRollupAddress(),