diff --git a/docs/docs/developers/migration_notes.md b/docs/docs/developers/migration_notes.md index 905f628b4c9c..465d520e1291 100644 --- a/docs/docs/developers/migration_notes.md +++ b/docs/docs/developers/migration_notes.md @@ -12,6 +12,10 @@ Aztec is in full-speed development. Literally every version breaks compatibility This release includes a major architectural change to the system. The PXE JSON RPC Server has been removed, and PXE is now available only as a library to be used by wallets. +## [Aztec node] + +Network config. The node now pulls default configuration from the public repository [AztecProtocol/networks](https://github.com/AztecProtocol/networks) after it applies the configuration it takes from the running environment and the configuration values baked into the source code. See associated [Design document](https://github.com/AztecProtocol/engineering-designs/blob/15415a62a7c8e901acb8e523625e91fc6f71dce4/docs/network-config/dd.md) + ## [Aztec.js] ### CLI Wallet commands dropped from `aztec` command diff --git a/yarn-project/aztec/src/bin/index.ts b/yarn-project/aztec/src/bin/index.ts index 99508283ba92..10cde18ec30b 100644 --- a/yarn-project/aztec/src/bin/index.ts +++ b/yarn-project/aztec/src/bin/index.ts @@ -2,7 +2,7 @@ // import { injectCommands as injectBuilderCommands } from '@aztec/builder'; import { injectCommands as injectAztecNodeCommands } from '@aztec/cli/aztec_node'; -import { enrichEnvironmentWithChainConfig } from '@aztec/cli/config'; +import { enrichEnvironmentWithChainConfig, enrichEnvironmentWithNetworkConfig } from '@aztec/cli/config'; import { injectCommands as injectContractCommands } from '@aztec/cli/contracts'; import { injectCommands as injectDevnetCommands } from '@aztec/cli/devnet'; import { injectCommands as injectInfrastructureCommands } from '@aztec/cli/infrastructure'; @@ -38,7 +38,9 @@ async function main() { networkValue = args[networkIndex].split('=')[1] || args[networkIndex + 1]; } - await enrichEnvironmentWithChainConfig(getActiveNetworkName(networkValue)); + const networkName = getActiveNetworkName(networkValue); + await enrichEnvironmentWithChainConfig(networkName); + await enrichEnvironmentWithNetworkConfig(networkName); const cliVersion = getCliVersion(); let program = new Command('aztec'); diff --git a/yarn-project/cli/src/config/cached_fetch.ts b/yarn-project/cli/src/config/cached_fetch.ts new file mode 100644 index 000000000000..26dde5c735cc --- /dev/null +++ b/yarn-project/cli/src/config/cached_fetch.ts @@ -0,0 +1,67 @@ +import { createLogger } from '@aztec/aztec.js'; + +import { mkdir, readFile, stat, writeFile } from 'fs/promises'; +import { dirname } from 'path'; + +export interface CachedFetchOptions { + /** Cache duration in milliseconds */ + cacheDurationMs: number; + /** The cache file */ + cacheFile?: string; +} + +/** + * Fetches data from a URL with file-based caching support. + * This utility can be used by both remote config and bootnodes fetching. + * + * @param url - The URL to fetch from + * @param networkName - Network name for cache directory structure + * @param options - Caching and error handling options + * @param cacheDir - Optional cache directory (defaults to no caching) + * @returns The fetched and parsed JSON data, or undefined if fetch fails and throwOnError is false + */ +export async function cachedFetch( + url: string, + options: CachedFetchOptions, + fetch = globalThis.fetch, + log = createLogger('cached_fetch'), +): Promise { + const { cacheDurationMs, cacheFile } = options; + + // Try to read from cache first + try { + if (cacheFile) { + const info = await stat(cacheFile); + if (info.mtimeMs + cacheDurationMs > Date.now()) { + const cachedData = JSON.parse(await readFile(cacheFile, 'utf-8')); + return cachedData; + } + } + } catch { + log.trace('Failed to read data from cache'); + } + + try { + const response = await fetch(url); + if (!response.ok) { + log.warn(`Failed to fetch from ${url}: ${response.status} ${response.statusText}`); + return undefined; + } + + const data = await response.json(); + + try { + if (cacheFile) { + await mkdir(dirname(cacheFile), { recursive: true }); + await writeFile(cacheFile, JSON.stringify(data), 'utf-8'); + } + } catch (err) { + log.warn('Failed to cache data on disk: ' + cacheFile, { cacheFile, err }); + } + + return data; + } catch (err) { + log.warn(`Failed to fetch from ${url}`, { err }); + return undefined; + } +} diff --git a/yarn-project/cli/src/config/chain_l2_config.ts b/yarn-project/cli/src/config/chain_l2_config.ts index cdcbb2ce4730..c8e987a1d918 100644 --- a/yarn-project/cli/src/config/chain_l2_config.ts +++ b/yarn-project/cli/src/config/chain_l2_config.ts @@ -4,10 +4,11 @@ import { EthAddress } from '@aztec/foundation/eth-address'; import type { SharedNodeConfig } from '@aztec/node-lib/config'; import type { SlasherConfig } from '@aztec/stdlib/interfaces/server'; -import { mkdir, readFile, stat, writeFile } from 'fs/promises'; -import path, { dirname, join } from 'path'; +import path, { join } from 'path'; import publicIncludeMetrics from '../../public_include_metric_prefixes.json' with { type: 'json' }; +import { cachedFetch } from './cached_fetch.js'; +import { enrichEthAddressVar, enrichVar } from './enrich_env.js'; const SNAPSHOT_URL = 'https://pub-f4a8c34d4bb7441ebf8f48d904512180.r2.dev/snapshots'; @@ -79,9 +80,9 @@ export const stagingIgnitionL2ChainConfig: L2ChainConfig = { sponsoredFPC: false, p2pEnabled: true, p2pBootstrapNodes: [], - registryAddress: '0x5f85fa0f40bc4b5ccd53c9f34258aa55d25cdde8', - slashFactoryAddress: '0x257db2ca1471b7f76f414d2997404bfbe916c8c9', - feeAssetHandlerAddress: '0x67d645b0a3e053605ea861d7e8909be6669812c4', + registryAddress: '0x53e2c2148da04fd0e8dd282f016f627a187c292c', + slashFactoryAddress: '0x56448efb139ef440438dfa445333734ea5ed60a0', + feeAssetHandlerAddress: '0x3613b834544030c166a4d47eca14b910f4816f57', seqMinTxsPerBlock: 0, seqMaxTxsPerBlock: 0, realProofs: true, @@ -160,9 +161,9 @@ export const stagingPublicL2ChainConfig: L2ChainConfig = { sponsoredFPC: true, p2pEnabled: true, p2pBootstrapNodes: [], - registryAddress: '0x2e48addca360da61e4d6c21ff2b1961af56eb83b', - slashFactoryAddress: '0xe19410632fd00695bc5a08dd82044b7b26317742', - feeAssetHandlerAddress: '0xb46dc3d91f849999330b6dd93473fa29fc45b076', + registryAddress: '0xe83067689f3cf837ccbf8a3966f0e0fe985dcb3e', + slashFactoryAddress: '0x8b87a1812162d4890f01bb40f410047f37d3ceb8', + feeAssetHandlerAddress: '0xa8159159a9e2a57c6e8c59fd5b3dd94c6dbddfe3', seqMinTxsPerBlock: 0, seqMaxTxsPerBlock: 20, realProofs: true, @@ -265,37 +266,13 @@ export const testnetL2ChainConfig: L2ChainConfig = { const BOOTNODE_CACHE_DURATION_MS = 60 * 60 * 1000; // 1 hour; export async function getBootnodes(networkName: NetworkNames, cacheDir?: string) { - const cacheFile = cacheDir ? join(cacheDir, networkName, 'bootnodes.json') : undefined; - try { - if (cacheFile) { - const info = await stat(cacheFile); - if (info.mtimeMs + BOOTNODE_CACHE_DURATION_MS > Date.now()) { - return JSON.parse(await readFile(cacheFile, 'utf-8'))['bootnodes']; - } - } - } catch { - // no-op. Get the remote-file - } - const url = `http://static.aztec.network/${networkName}/bootnodes.json`; - const response = await fetch(url); - if (!response.ok) { - throw new Error( - `Failed to fetch basic contract addresses from ${url}. Check you are using a correct network name.`, - ); - } - const json = await response.json(); - - try { - if (cacheFile) { - await mkdir(dirname(cacheFile), { recursive: true }); - await writeFile(cacheFile, JSON.stringify(json), 'utf-8'); - } - } catch { - // no-op - } + const data = await cachedFetch(url, { + cacheDurationMs: BOOTNODE_CACHE_DURATION_MS, + cacheFile: cacheDir ? join(cacheDir, networkName, 'bootnodes.json') : undefined, + }); - return json['bootnodes']; + return data?.bootnodes; } export async function getL2ChainConfig( @@ -321,23 +298,6 @@ export async function getL2ChainConfig( return config; } -function enrichVar(envVar: EnvVar, value: string | undefined) { - // Don't override - if (process.env[envVar] || value === undefined) { - return; - } - process.env[envVar] = value; -} - -function enrichEthAddressVar(envVar: EnvVar, value: string) { - // EthAddress doesn't like being given empty strings - if (value === '') { - enrichVar(envVar, EthAddress.ZERO.toString()); - return; - } - enrichVar(envVar, value); -} - function getDefaultDataDir(networkName: NetworkNames): string { return path.join(process.env.HOME || '~', '.aztec', networkName, 'data'); } diff --git a/yarn-project/cli/src/config/enrich_env.ts b/yarn-project/cli/src/config/enrich_env.ts new file mode 100644 index 000000000000..4712157859ba --- /dev/null +++ b/yarn-project/cli/src/config/enrich_env.ts @@ -0,0 +1,15 @@ +import { EthAddress } from '@aztec/aztec.js'; +import type { EnvVar } from '@aztec/foundation/config'; + +export function enrichVar(envVar: EnvVar, value: string | undefined) { + // Don't override + if (process.env[envVar] || value === undefined) { + return; + } + process.env[envVar] = value; +} + +export function enrichEthAddressVar(envVar: EnvVar, value: string) { + // EthAddress doesn't like being given empty strings + enrichVar(envVar, value || EthAddress.ZERO.toString()); +} diff --git a/yarn-project/cli/src/config/index.ts b/yarn-project/cli/src/config/index.ts index 5e6e849aa628..f421f72041e3 100644 --- a/yarn-project/cli/src/config/index.ts +++ b/yarn-project/cli/src/config/index.ts @@ -1,2 +1,4 @@ +export * from './cached_fetch.js'; export * from './chain_l2_config.js'; export * from './get_l1_config.js'; +export * from './network_config.js'; diff --git a/yarn-project/cli/src/config/network_config.ts b/yarn-project/cli/src/config/network_config.ts new file mode 100644 index 000000000000..8c1e33834388 --- /dev/null +++ b/yarn-project/cli/src/config/network_config.ts @@ -0,0 +1,108 @@ +import { type NetworkConfig, NetworkConfigMapSchema, type NetworkNames } from '@aztec/foundation/config'; + +import { readFile } from 'fs/promises'; +import { join } from 'path'; + +import { cachedFetch } from './cached_fetch.js'; +import { enrichEthAddressVar, enrichVar } from './enrich_env.js'; + +const DEFAULT_CONFIG_URL = + 'https://raw.githubusercontent.com/AztecProtocol/networks/refs/heads/main/network_config.json'; +const NETWORK_CONFIG_CACHE_DURATION_MS = 60 * 60 * 1000; // 1 hour + +/** + * Fetches remote network configuration from GitHub with caching support. + * Uses the reusable cachedFetch utility. + * + * @param networkName - The network name to fetch config for + * @param cacheDir - Optional cache directory for storing fetched config + * @returns Remote configuration for the specified network, or undefined if not found/error + */ +export async function getNetworkConfig( + networkName: NetworkNames, + cacheDir?: string, +): Promise { + let url: URL | undefined; + const configLocation = process.env.NETWORK_CONFIG_LOCATION || DEFAULT_CONFIG_URL; + + if (!configLocation) { + return undefined; + } + + try { + if (configLocation.includes('://')) { + url = new URL(configLocation); + } else { + url = new URL(`file://${configLocation}`); + } + } catch { + /* no-op */ + } + + if (!url) { + return undefined; + } + + try { + let rawConfig: any; + + if (url.protocol === 'http:' || url.protocol === 'https:') { + rawConfig = await cachedFetch(url.href, { + cacheDurationMs: NETWORK_CONFIG_CACHE_DURATION_MS, + cacheFile: cacheDir ? join(cacheDir, networkName, 'network_config.json') : undefined, + }); + } else if (url.protocol === 'file:') { + rawConfig = JSON.parse(await readFile(url.pathname, 'utf-8')); + } else { + throw new Error('Unsupported Aztec network config protocol: ' + url.href); + } + + if (!rawConfig) { + return undefined; + } + + const networkConfigMap = NetworkConfigMapSchema.parse(rawConfig); + if (networkName in networkConfigMap) { + return networkConfigMap[networkName]; + } else { + return undefined; + } + } catch { + return undefined; + } +} + +/** + * Enriches environment variables with remote network configuration. + * This function is called before node config initialization to set env vars + * from the remote config, following the same pattern as enrichEnvironmentWithChainConfig(). + * + * @param networkName - The network name to fetch remote config for + */ +export async function enrichEnvironmentWithNetworkConfig(networkName: NetworkNames) { + if (networkName === 'local') { + return; // No remote config for local development + } + + const cacheDir = process.env.DATA_DIRECTORY ? join(process.env.DATA_DIRECTORY, 'cache') : undefined; + const networkConfig = await getNetworkConfig(networkName, cacheDir); + + if (!networkConfig) { + return; + } + + enrichVar('BOOTSTRAP_NODES', networkConfig.bootnodes.join(',')); + enrichVar('L1_CHAIN_ID', String(networkConfig.l1ChainId)); + + // Snapshot synch only supports a single source. Take the first + // See A-101 for more details + const firstSource = networkConfig[0]; + if (firstSource) { + enrichVar('SYNC_SNAPSHOTS_URL', networkConfig.snapshots.join(',')); + } + + enrichEthAddressVar('REGISTRY_CONTRACT_ADDRESS', networkConfig.registryAddress.toString()); + if (networkConfig.feeAssetHandlerAddress) { + enrichEthAddressVar('FEE_ASSET_HANDLER_CONTRACT_ADDRESS', networkConfig.feeAssetHandlerAddress.toString()); + } +} diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 4803597e7cb7..f40ccc154787 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -77,6 +77,7 @@ export type EnvVar = | 'LOG_LEVEL' | 'MNEMONIC' | 'NETWORK' + | 'NETWORK_CONFIG_LOCATION' | 'NO_PXE' | 'USE_GCLOUD_LOGGING' | 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT' diff --git a/yarn-project/foundation/src/config/index.ts b/yarn-project/foundation/src/config/index.ts index d195f6b70b48..65b6428b8997 100644 --- a/yarn-project/foundation/src/config/index.ts +++ b/yarn-project/foundation/src/config/index.ts @@ -5,6 +5,8 @@ import { SecretValue } from './secret_value.js'; export { SecretValue, getActiveNetworkName }; export type { EnvVar, NetworkNames }; +export type { NetworkConfig, NetworkConfigMap } from './network_config.js'; +export { NetworkConfigMapSchema, NetworkConfigSchema } from './network_config.js'; export interface ConfigMapping { env?: EnvVar; diff --git a/yarn-project/foundation/src/config/network_config.test.ts b/yarn-project/foundation/src/config/network_config.test.ts new file mode 100644 index 000000000000..ca484686d25a --- /dev/null +++ b/yarn-project/foundation/src/config/network_config.test.ts @@ -0,0 +1,150 @@ +import { NetworkConfigMapSchema, NetworkConfigSchema } from './network_config.js'; + +describe('NetworkConfig', () => { + describe('NetworkConfigSchema', () => { + it('should validate a valid remote config', () => { + const validConfigInput = { + bootnodes: ['enr:-test1', 'enr:-test2'], + snapshots: ['https://example.com/snapshot1.tar.gz'], + registryAddress: '0x1234567890123456789012345678901234567890', + feeAssetHandlerAddress: '0x2345678901234567890123456789012345678901', + l1ChainId: 11155111, + }; + + const result = NetworkConfigSchema.safeParse(validConfigInput); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.bootnodes).toEqual(validConfigInput.bootnodes); + expect(result.data.snapshots).toEqual(validConfigInput.snapshots); + expect(result.data.registryAddress.toString()).toBe(validConfigInput.registryAddress); + expect(result.data.feeAssetHandlerAddress?.toString()).toBe(validConfigInput.feeAssetHandlerAddress); + expect(result.data.l1ChainId).toBe(validConfigInput.l1ChainId); + } + }); + + it('should validate config without optional feeAssetHandlerAddress', () => { + const validConfig = { + bootnodes: ['enr:-test1'], + snapshots: ['https://example.com/snapshot1.tar.gz'], + registryAddress: '0x1234567890123456789012345678901234567890', + l1ChainId: 11155111, + }; + + const result = NetworkConfigSchema.safeParse(validConfig); + expect(result.success).toBe(true); + }); + + it('should reject invalid config with missing required fields', () => { + const invalidConfig = { + bootnodes: ['enr:-test1'], + // Missing required fields + }; + + const result = NetworkConfigSchema.safeParse(invalidConfig); + expect(result.success).toBe(false); + }); + + it('should allow additional unknown fields (permissive parsing)', () => { + const configWithExtraFields = { + bootnodes: ['enr:-test1'], + snapshots: ['https://example.com/snapshot1.tar.gz'], + registryAddress: '0x1234567890123456789012345678901234567890', + l1ChainId: 11155111, + // New fields that might be added in the future + newFeature: 'enabled', + futureConfig: { + someNestedValue: 42, + anotherValue: 'test', + }, + arrayOfNewStuff: ['item1', 'item2'], + }; + + const result = NetworkConfigSchema.safeParse(configWithExtraFields); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.bootnodes).toEqual(configWithExtraFields.bootnodes); + expect(result.data.registryAddress.toString()).toBe(configWithExtraFields.registryAddress); + expect(result.data.l1ChainId).toBe(configWithExtraFields.l1ChainId); + // Verify that unknown fields are preserved + expect((result.data as any).newFeature).toBe('enabled'); + expect((result.data as any).futureConfig).toEqual(configWithExtraFields.futureConfig); + expect((result.data as any).arrayOfNewStuff).toEqual(configWithExtraFields.arrayOfNewStuff); + } + }); + }); + + describe('NetworkConfigMapSchema', () => { + it('should validate multiple network configurations', () => { + const networkConfigInput = { + 'staging-public': { + bootnodes: ['enr:-staging1'], + snapshots: ['https://example.com/staging-snapshot.tar.gz'], + registryAddress: '0x1234567890123456789012345678901234567890', + l1ChainId: 11155111, + }, + testnet: { + bootnodes: ['enr:-testnet1', 'enr:-testnet2'], + snapshots: ['https://example.com/testnet-snapshot.tar.gz'], + registryAddress: '0x2345678901234567890123456789012345678901', + feeAssetHandlerAddress: '0x3456789012345678901234567890123456789012', + l1ChainId: 1, + }, + }; + + const result = NetworkConfigMapSchema.safeParse(networkConfigInput); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data['staging-public'].registryAddress.toString()).toBe( + '0x1234567890123456789012345678901234567890', + ); + expect(result.data['testnet'].registryAddress.toString()).toBe('0x2345678901234567890123456789012345678901'); + expect(result.data['testnet'].feeAssetHandlerAddress?.toString()).toBe( + '0x3456789012345678901234567890123456789012', + ); + } + }); + + it('should handle future network config schema evolution', () => { + const futureFriendlyNetworkConfig = { + 'staging-public': { + bootnodes: ['enr:-staging1'], + snapshots: ['https://example.com/staging-snapshot.tar.gz'], + registryAddress: '0x1234567890123456789012345678901234567890', + l1ChainId: 11155111, + // Future fields that don't exist in current schema + newBootnodeFormat: ['multiaddr:/ip4/...'], + advancedP2PConfig: { + maxPeers: 50, + timeout: 30000, + }, + }, + testnet: { + bootnodes: ['enr:-testnet1'], + snapshots: ['https://example.com/testnet-snapshot.tar.gz'], + registryAddress: '0x2345678901234567890123456789012345678901', + l1ChainId: 1, + // Different future fields per network + experimentalFeatures: ['feature1', 'feature2'], + }, + }; + + const result = NetworkConfigMapSchema.safeParse(futureFriendlyNetworkConfig); + expect(result.success).toBe(true); + if (result.success) { + // Verify existing fields still work + expect(result.data['staging-public'].registryAddress.toString()).toBe( + '0x1234567890123456789012345678901234567890', + ); + expect(result.data['testnet'].registryAddress.toString()).toBe('0x2345678901234567890123456789012345678901'); + + // Verify future fields are preserved + expect((result.data['staging-public'] as any).newBootnodeFormat).toEqual(['multiaddr:/ip4/...']); + expect((result.data['staging-public'] as any).advancedP2PConfig).toEqual({ + maxPeers: 50, + timeout: 30000, + }); + expect((result.data['testnet'] as any).experimentalFeatures).toEqual(['feature1', 'feature2']); + } + }); + }); +}); diff --git a/yarn-project/foundation/src/config/network_config.ts b/yarn-project/foundation/src/config/network_config.ts new file mode 100644 index 000000000000..97d976a6de2e --- /dev/null +++ b/yarn-project/foundation/src/config/network_config.ts @@ -0,0 +1,16 @@ +import { z } from 'zod'; + +export const NetworkConfigSchema = z + .object({ + bootnodes: z.array(z.string()), + snapshots: z.array(z.string()), + registryAddress: z.string(), + feeAssetHandlerAddress: z.string().optional(), + l1ChainId: z.number(), + }) + .passthrough(); // Allow additional unknown fields to pass through + +export const NetworkConfigMapSchema = z.record(z.string(), NetworkConfigSchema); + +export type NetworkConfig = z.infer; +export type NetworkConfigMap = z.infer; diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index ec82bb0aaacb..5ab78914925b 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -54,6 +54,13 @@ export async function createP2PClient( }); const logger = deps.logger ?? createLogger('p2p'); + + if (config.bootstrapNodes.length === 0) { + logger.warn( + 'No bootstrap nodes have been provided. Set the BOOTSTRAP_NODES environment variable in order to join the P2P network', + ); + } + const store = deps.store ?? (await createStore(P2P_STORE_NAME, 2, config, createLogger('p2p:lmdb-v2'))); const archive = await createStore(P2P_ARCHIVE_STORE_NAME, 1, config, createLogger('p2p-archive:lmdb-v2')); const peerStore = await createStore(P2P_PEER_STORE_NAME, 1, config, createLogger('p2p-peer:lmdb-v2'));