diff --git a/yarn-project/archiver/src/archiver/config.ts b/yarn-project/archiver/src/archiver/config.ts index 6be9a705ff5f..8fc408b04cf5 100644 --- a/yarn-project/archiver/src/archiver/config.ts +++ b/yarn-project/archiver/src/archiver/config.ts @@ -40,6 +40,9 @@ export type ArchiverConfig = { /** The max number of logs that can be obtained in 1 "getPublicLogs" call. */ maxLogs?: number; + + /** The maximum possible size of the archiver DB in KB. Overwrites the general dataStoreMapSizeKB. */ + archiverStoreMapSizeKb?: number; } & L1ReaderConfig & L1ContractsConfig & BlobSinkConfig & @@ -72,6 +75,11 @@ export const archiverConfigMappings: ConfigMappingsType = { description: 'The max number of logs that can be obtained in 1 "getPublicLogs" call.', ...numberConfigHelper(1_000), }, + archiverStoreMapSizeKb: { + env: 'ARCHIVER_STORE_MAP_SIZE_KB', + parseEnv: (val: string | undefined) => (val ? +val : undefined), + description: 'The maximum possible size of the archiver DB in KB. Overwrites the general dataStoreMapSizeKB.', + }, ...chainConfigMappings, ...l1ReaderConfigMappings, viemPollingIntervalMS: { diff --git a/yarn-project/archiver/src/factory.ts b/yarn-project/archiver/src/factory.ts index cffc08d05f9a..39abdbcbc28f 100644 --- a/yarn-project/archiver/src/factory.ts +++ b/yarn-project/archiver/src/factory.ts @@ -26,11 +26,12 @@ import { createArchiverClient } from './rpc/index.js'; * @returns The local archiver. */ export async function createArchiver( - config: ArchiverConfig & DataStoreConfig, + _config: ArchiverConfig & DataStoreConfig, blobSinkClient: BlobSinkClientInterface, opts: { blockUntilSync: boolean } = { blockUntilSync: true }, telemetry: TelemetryClient = getTelemetryClient(), ): Promise { + const config = { ..._config, dataStoreMapSizeKB: _config.archiverStoreMapSizeKb ?? _config.dataStoreMapSizeKB }; const store = await createStore( 'archiver', KVArchiverDataStore.SCHEMA_VERSION, diff --git a/yarn-project/aztec/src/cli/aztec_start_options.ts b/yarn-project/aztec/src/cli/aztec_start_options.ts index dd0461414d11..ee7e435252f2 100644 --- a/yarn-project/aztec/src/cli/aztec_start_options.ts +++ b/yarn-project/aztec/src/cli/aztec_start_options.ts @@ -62,6 +62,7 @@ export const universalOptions = [ 'l1Contracts', 'p2pEnabled', 'dataDirectory', + 'dataStoreMapSizeKb', ]; // Define categories and options @@ -157,6 +158,22 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = { envVar: 'L1_CONSENSUS_HOST_API_KEY_HEADER', }, ], + STORAGE: [ + { + flag: '--data-directory ', + description: 'Where to store data for services. If not set, will store temporarily', + defaultValue: undefined, + envVar: 'DATA_DIRECTORY', + }, + { + flag: '--data-store-map-size-kb ', + description: + 'The maximum possible size of the data store DB in KB. Can be overridden by component-specific options.', + defaultValue: undefined, + envVar: 'DATA_STORE_MAP_SIZE_KB', + parseVal: (val: string) => parseInt(val, 10), + }, + ], 'L1 CONTRACT ADDRESSES': [ { flag: '--rollup-address ', @@ -209,12 +226,6 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = { defaultValue: undefined, envVar: undefined, }, - { - flag: '--data-directory ', - description: 'Where to store data. If not set, will store temporarily', - defaultValue: undefined, - envVar: 'DATA_DIRECTORY', - }, { flag: '--node.archiverUrl ', description: 'URL for an archiver service', @@ -301,7 +312,7 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = { }, ...getOptions('sequencer', sequencerClientConfigMappings), ], - BLOB_SINK: [ + 'BLOB SINK': [ { flag: '--blob-sink', description: 'Starts Aztec Blob Sink with options', diff --git a/yarn-project/aztec/src/cli/cmds/start_archiver.ts b/yarn-project/aztec/src/cli/cmds/start_archiver.ts index 59fb8e62a16f..f17a29a345b3 100644 --- a/yarn-project/aztec/src/cli/cmds/start_archiver.ts +++ b/yarn-project/aztec/src/cli/cmds/start_archiver.ts @@ -37,6 +37,7 @@ export async function startArchiver( ); let archiverConfig = { ...envConfig, ...cliOptions }; + archiverConfig.dataStoreMapSizeKB = archiverConfig.archiverStoreMapSizeKb ?? archiverConfig.dataStoreMapSizeKB; if (!archiverConfig.l1Contracts.registryAddress || archiverConfig.l1Contracts.registryAddress.isZero()) { throw new Error('L1 registry address is required to start an Archiver'); diff --git a/yarn-project/blob-sink/src/server/config.ts b/yarn-project/blob-sink/src/server/config.ts index a5801c66001e..0ea9d1bcd7ee 100644 --- a/yarn-project/blob-sink/src/server/config.ts +++ b/yarn-project/blob-sink/src/server/config.ts @@ -11,22 +11,21 @@ import { type BlobSinkArchiveApiConfig, blobSinkArchiveApiConfigMappings } from export type BlobSinkConfig = { port?: number; - dataStoreConfig?: DataStoreConfig; + archiveApiUrl?: string; } & BlobSinkArchiveApiConfig & - Partial & Pick>; + Partial & Pick> & + Partial; export const blobSinkConfigMappings: ConfigMappingsType = { port: { env: 'BLOB_SINK_PORT', description: 'The port to run the blob sink server on', }, - dataStoreConfig: { - ...dataConfigMappings, - description: 'The configuration for the data store', - }, + ...blobSinkArchiveApiConfigMappings, ...pickConfigMappings(l1ReaderConfigMappings, ['l1RpcUrls']), ...pickConfigMappings(l1ContractAddressesMapping, ['rollupAddress']), + ...dataConfigMappings, }; /** diff --git a/yarn-project/blob-sink/src/server/factory.ts b/yarn-project/blob-sink/src/server/factory.ts index 40ef2a6b7a70..8b2f19de23fa 100644 --- a/yarn-project/blob-sink/src/server/factory.ts +++ b/yarn-project/blob-sink/src/server/factory.ts @@ -9,21 +9,26 @@ import { BlobSinkServer } from './server.js'; // If data store settings are provided, the store is created and returned. // Otherwise, undefined is returned and an in memory store will be used. -async function getDataStoreConfig(config?: BlobSinkConfig): Promise { - if (!config?.dataStoreConfig) { +async function getDataStore(config?: BlobSinkConfig): Promise { + if (!config?.dataDirectory || !config?.dataStoreMapSizeKB) { return undefined; } - return await createStore('blob-sink', 1, config.dataStoreConfig); + return await createStore('blob-sink', 1, { + ...config, + // re-assigning to make TypeScript happy + dataDirectory: config.dataDirectory, + dataStoreMapSizeKB: config.dataStoreMapSizeKB, + }); } /** * Creates a blob sink service from the provided config. */ export async function createBlobSinkServer( - config: BlobSinkConfig = {}, + config: BlobSinkConfig, telemetry?: TelemetryClient, ): Promise { - const store = await getDataStoreConfig(config); + const store = await getDataStore(config); const archiveClient = createBlobArchiveClient(config); const { l1ChainId, l1RpcUrls } = config; const l1PublicClient = l1ChainId && l1RpcUrls ? getPublicClient({ l1ChainId, l1RpcUrls }) : undefined; diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index 214c9eb4daa8..5d37c9505480 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -403,10 +403,8 @@ async function setupFromFresh( l1RpcUrls: aztecNodeConfig.l1RpcUrls, rollupAddress: aztecNodeConfig.l1Contracts.rollupAddress, port: blobSinkPort, - dataStoreConfig: { - dataDirectory: aztecNodeConfig.dataDirectory, - dataStoreMapSizeKB: aztecNodeConfig.dataStoreMapSizeKB, - }, + dataDirectory: aztecNodeConfig.dataDirectory, + dataStoreMapSizeKB: aztecNodeConfig.dataStoreMapSizeKB, }, telemetry, ); @@ -525,10 +523,8 @@ async function setupFromState(statePath: string, logger: Logger): Promise = { }, l1Contracts: { description: 'The deployed L1 contract addresses', - nested: l1ContractAddressesMapping, + nested: { + rollupAddress: l1ContractAddressesMapping.rollupAddress, + }, }, }; diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index 815429776727..cfbffd945bdb 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -36,7 +36,7 @@ export const createP2PClient = async ( telemetry: TelemetryClient = getTelemetryClient(), deps: P2PClientDeps = {}, ) => { - let config = { ..._config }; + let config = { ..._config, dataStoreMapSizeKB: _config.p2pStoreMapSizeKb ?? _config.dataStoreMapSizeKB }; const logger = deps.logger ?? createLogger('p2p'); const store = deps.store ?? (await createStore('p2p', 1, config, createLogger('p2p:lmdb-v2'))); const archive = await createStore('p2p-archive', 1, config, createLogger('p2p-archive:lmdb-v2')); diff --git a/yarn-project/p2p/src/config.ts b/yarn-project/p2p/src/config.ts index fe2995653aa7..b4f3f91bd033 100644 --- a/yarn-project/p2p/src/config.ts +++ b/yarn-project/p2p/src/config.ts @@ -164,6 +164,11 @@ export interface P2PConfig extends P2PReqRespConfig, ChainConfig { * A list of trusted peers. */ trustedPeers: string[]; + + /** + * The maximum possible size of the P2P DB in KB. Overwrites the general dataStoreMapSizeKB. + */ + p2pStoreMapSizeKb?: number; } export const p2pConfigMappings: ConfigMappingsType = { @@ -331,6 +336,11 @@ export const p2pConfigMappings: ConfigMappingsType = { description: 'A list of trusted peers ENRs. Separated by commas.', defaultValue: [], }, + p2pStoreMapSizeKb: { + env: 'P2P_STORE_MAP_SIZE_KB', + parseEnv: (val: string | undefined) => (val ? +val : undefined), + description: 'The maximum possible size of the P2P DB in KB. Overwrites the general dataStoreMapSizeKB.', + }, ...p2pReqRespConfigMappings, ...chainConfigMappings, }; diff --git a/yarn-project/prover-client/src/proving_broker/config.ts b/yarn-project/prover-client/src/proving_broker/config.ts index 3e89e0e61e1e..aca7a1b61770 100644 --- a/yarn-project/prover-client/src/proving_broker/config.ts +++ b/yarn-project/prover-client/src/proving_broker/config.ts @@ -21,6 +21,8 @@ export const ProverBrokerConfig = z.object({ dataDirectory: z.string().optional(), /** The size of the data store map */ dataStoreMapSizeKB: z.number().int().nonnegative(), + /** The size of the prover broker's database. Will override the dataStoreMapSizeKB if set. */ + proverBrokerStoreMapSizeKB: z.number().int().nonnegative().optional(), /** The prover broker may batch jobs together before writing to the database */ proverBrokerBatchSize: z.number().int().nonnegative(), /** How often the job batches get flushed */ @@ -64,6 +66,11 @@ export const proverBrokerConfigMappings: ConfigMappingsType description: 'The maximum number of epochs to keep results for', ...numberConfigHelper(1), }, + proverBrokerStoreMapSizeKB: { + env: 'PROVER_BROKER_STORE_MAP_SIZE_KB', + parseEnv: (val: string | undefined) => (val ? +val : undefined), + description: "The size of the prover broker's database. Will override the dataStoreMapSizeKB if set.", + }, ...l1ReaderConfigMappings, ...dataConfigMappings, }; diff --git a/yarn-project/prover-client/src/proving_broker/factory.ts b/yarn-project/prover-client/src/proving_broker/factory.ts index 367398f86ee9..5c236dcbc9e1 100644 --- a/yarn-project/prover-client/src/proving_broker/factory.ts +++ b/yarn-project/prover-client/src/proving_broker/factory.ts @@ -6,9 +6,10 @@ import { InMemoryBrokerDatabase } from './proving_broker_database/memory.js'; import { KVBrokerDatabase } from './proving_broker_database/persisted.js'; export async function createAndStartProvingBroker( - config: ProverBrokerConfig, + _config: ProverBrokerConfig, client: TelemetryClient, ): Promise { + const config = { ..._config, dataStoreMapSizeKB: _config.proverBrokerStoreMapSizeKB ?? _config.dataStoreMapSizeKB }; const database = config.dataDirectory ? await KVBrokerDatabase.new(config, client) : new InMemoryBrokerDatabase(); const broker = new ProvingBroker(database, config, client); diff --git a/yarn-project/world-state/src/synchronizer/config.ts b/yarn-project/world-state/src/synchronizer/config.ts index 3714126cb5c8..b547f3d1e122 100644 --- a/yarn-project/world-state/src/synchronizer/config.ts +++ b/yarn-project/world-state/src/synchronizer/config.ts @@ -46,7 +46,7 @@ export const worldStateConfigMappings: ConfigMappingsType = { worldStateDbMapSizeKb: { env: 'WS_DB_MAP_SIZE_KB', parseEnv: (val: string | undefined) => (val ? +val : undefined), - description: 'The maximum possible size of the world state DB', + description: 'The maximum possible size of the world state DB in KB. Overwrites the general dataStoreMapSizeKB.', }, worldStateDataDirectory: { env: 'WS_DATA_DIRECTORY',