chore: allow individual service data map size configuration

yarn-project/archiver/src/archiver/config.ts

@@ -40,6 +40,9 @@ export type ArchiverConfig = {
 
   /** The max number of logs that can be obtained in 1 "getPublicLogs" call. */
   maxLogs?: number;
+
+  /** The maximum possible size of the archiver DB in KB. Overwrites the general dataStoreMapSizeKB. */
+  archiverStoreMapSizeKb?: number;
 } & L1ReaderConfig &
   L1ContractsConfig &
   BlobSinkConfig &
@@ -72,6 +75,11 @@ export const archiverConfigMappings: ConfigMappingsType<ArchiverConfig> = {
     description: 'The max number of logs that can be obtained in 1 "getPublicLogs" call.',
     ...numberConfigHelper(1_000),
   },
+  archiverStoreMapSizeKb: {
+    env: 'ARCHIVER_STORE_MAP_SIZE_KB',
+    parseEnv: (val: string | undefined) => (val ? +val : undefined),
+    description: 'The maximum possible size of the archiver DB in KB. Overwrites the general dataStoreMapSizeKB.',
+  },
   ...chainConfigMappings,
   ...l1ReaderConfigMappings,
   viemPollingIntervalMS: {

yarn-project/archiver/src/factory.ts

@@ -32,11 +32,12 @@ import { createArchiverClient } from './rpc/index.js';
  * @returns The local archiver.
  */
 export async function createArchiver(
-  config: ArchiverConfig & DataStoreConfig,
+  _config: ArchiverConfig & DataStoreConfig,
   blobSinkClient: BlobSinkClientInterface,
   opts: { blockUntilSync: boolean } = { blockUntilSync: true },
   telemetry: TelemetryClient = getTelemetryClient(),
 ): Promise<ArchiverApi & Service & L2BlockSourceEventEmitter> {
+  const config = { ..._config, dataStoreMapSizeKB: _config.archiverStoreMapSizeKb ?? _config.dataStoreMapSizeKB };
   const store = await createStore(
     'archiver',
     KVArchiverDataStore.SCHEMA_VERSION,

yarn-project/aztec/src/cli/aztec_start_options.ts

@@ -62,6 +62,7 @@ export const universalOptions = [
   'l1Contracts',
   'p2pEnabled',
   'dataDirectory',
+  'dataStoreMapSizeKb',
 ];
 
 // Define categories and options
@@ -157,6 +158,22 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = {
       envVar: 'L1_CONSENSUS_HOST_API_KEY_HEADER',
     },
   ],
+  STORAGE: [
+    {
+      flag: '--data-directory <value>',
+      description: 'Where to store data for services. If not set, will store temporarily',
+      defaultValue: undefined,
+      envVar: 'DATA_DIRECTORY',
+    },
+    {
+      flag: '--data-store-map-size-kb <value>',
+      description:
+        'The maximum possible size of the data store DB in KB. Can be overridden by component-specific options.',
+      defaultValue: undefined,
+      envVar: 'DATA_STORE_MAP_SIZE_KB',
+      parseVal: (val: string) => parseInt(val, 10),
+    },
+  ],
   'L1 CONTRACT ADDRESSES': [
     {
       flag: '--rollup-address <value>',
@@ -209,12 +226,6 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = {
       defaultValue: undefined,
       envVar: undefined,
     },
-    {
-      flag: '--data-directory <value>',
-      description: 'Where to store data. If not set, will store temporarily',
-      defaultValue: undefined,
-      envVar: 'DATA_DIRECTORY',
-    },
     {
       flag: '--node.archiverUrl <value>',
       description: 'URL for an archiver service',
@@ -301,7 +312,7 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = {
     },
     ...getOptions('sequencer', sequencerClientConfigMappings),
   ],
-  BLOB_SINK: [
+  'BLOB SINK': [
     {
       flag: '--blob-sink',
       description: 'Starts Aztec Blob Sink with options',

yarn-project/aztec/src/cli/cmds/start_archiver.ts

@@ -37,6 +37,7 @@ export async function startArchiver(
   );
 
   let archiverConfig = { ...envConfig, ...cliOptions };
+  archiverConfig.dataStoreMapSizeKB = archiverConfig.archiverStoreMapSizeKb ?? archiverConfig.dataStoreMapSizeKB;
 
   if (!archiverConfig.l1Contracts.registryAddress || archiverConfig.l1Contracts.registryAddress.isZero()) {
     throw new Error('L1 registry address is required to start an Archiver');

yarn-project/blob-sink/src/server/config.ts

@@ -10,24 +10,21 @@ import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config
 export type BlobSinkConfig = {
   port?: number;
   archiveApiUrl?: string;
-  dataStoreConfig?: DataStoreConfig;
-} & Partial<Pick<L1ReaderConfig, 'l1ChainId' | 'l1RpcUrls'> & Pick<L1ContractAddresses, 'rollupAddress'>>;
+} & Partial<Pick<L1ReaderConfig, 'l1ChainId' | 'l1RpcUrls'> & Pick<L1ContractAddresses, 'rollupAddress'>> &
+  Partial<DataStoreConfig>;
 
 export const blobSinkConfigMappings: ConfigMappingsType<BlobSinkConfig> = {
   port: {
     env: 'BLOB_SINK_PORT',
     description: 'The port to run the blob sink server on',
   },
-  dataStoreConfig: {
-    ...dataConfigMappings,
-    description: 'The configuration for the data store',
-  },
   archiveApiUrl: {
     env: 'BLOB_SINK_ARCHIVE_API_URL',
     description: 'The URL of the archive API',
   },
   ...pickConfigMappings(l1ReaderConfigMappings, ['l1ChainId', 'l1RpcUrls']),
   ...pickConfigMappings(l1ContractAddressesMapping, ['rollupAddress']),
+  ...dataConfigMappings,
 };
 
 /**

yarn-project/blob-sink/src/server/factory.ts

@@ -9,21 +9,26 @@ import { BlobSinkServer } from './server.js';
 
 // If data store settings are provided, the store is created and returned.
 // Otherwise, undefined is returned and an in memory store will be used.
-async function getDataStoreConfig(config?: BlobSinkConfig): Promise<AztecAsyncKVStore | undefined> {
-  if (!config?.dataStoreConfig) {
+async function getDataStore(config?: BlobSinkConfig): Promise<AztecAsyncKVStore | undefined> {
+  if (!config?.dataDirectory || !config?.dataStoreMapSizeKB) {
     return undefined;
   }
-  return await createStore('blob-sink', 1, config.dataStoreConfig);
+  return await createStore('blob-sink', 1, {
+    ...config,
+    // re-assigning to make TypeScript happy
+    dataDirectory: config.dataDirectory,
+    dataStoreMapSizeKB: config.dataStoreMapSizeKB,
+  });
 }
 
 /**
  * Creates a blob sink service from the provided config.
  */
 export async function createBlobSinkServer(
-  config: BlobSinkConfig = {},
+  config: BlobSinkConfig,
   telemetry?: TelemetryClient,
 ): Promise<BlobSinkServer> {
-  const store = await getDataStoreConfig(config);
+  const store = await getDataStore(config);
   const archiveClient = createBlobArchiveClient(config);
   const { l1ChainId, l1RpcUrls } = config;
   const l1PublicClient = l1ChainId && l1RpcUrls ? getPublicClient({ l1ChainId, l1RpcUrls }) : undefined;

yarn-project/end-to-end/src/fixtures/snapshot_manager.ts

@@ -400,10 +400,8 @@ async function setupFromFresh(
       l1RpcUrls: aztecNodeConfig.l1RpcUrls,
       rollupAddress: aztecNodeConfig.l1Contracts.rollupAddress,
       port: blobSinkPort,
-      dataStoreConfig: {
-        dataDirectory: aztecNodeConfig.dataDirectory,
-        dataStoreMapSizeKB: aztecNodeConfig.dataStoreMapSizeKB,
-      },
+      dataDirectory: aztecNodeConfig.dataDirectory,
+      dataStoreMapSizeKB: aztecNodeConfig.dataStoreMapSizeKB,
     },
     telemetry,
   );
@@ -522,10 +520,8 @@ async function setupFromState(statePath: string, logger: Logger): Promise<Subsys
       l1RpcUrls: aztecNodeConfig.l1RpcUrls,
       rollupAddress: aztecNodeConfig.l1Contracts.rollupAddress,
       port: blobSinkPort,
-      dataStoreConfig: {
-        dataDirectory: statePath,
-        dataStoreMapSizeKB: aztecNodeConfig.dataStoreMapSizeKB,
-      },
+      dataDirectory: statePath,
+      dataStoreMapSizeKB: aztecNodeConfig.dataStoreMapSizeKB,
     },
     telemetry,
   );

yarn-project/end-to-end/src/fixtures/utils.ts

@@ -481,6 +481,8 @@ export async function setup(
       l1RpcUrls: config.l1RpcUrls,
       rollupAddress: config.l1Contracts.rollupAddress,
       port: blobSinkPort,
+      dataDirectory: config.dataDirectory,
+      dataStoreMapSizeKB: config.dataStoreMapSizeKB,
     },
     telemetry,
   );

yarn-project/foundation/src/config/env_var.ts

@@ -46,6 +46,10 @@ export type EnvVar =
   | 'COINBASE'
   | 'DATA_DIRECTORY'
   | 'DATA_STORE_MAP_SIZE_KB'
+  | 'ARCHIVER_STORE_MAP_SIZE_KB'
+  | 'P2P_STORE_MAP_SIZE_KB'
+  | 'PROVER_BROKER_STORE_MAP_SIZE_KB'
+  | 'WS_DB_MAP_SIZE_KB'
   | 'DEBUG'
   | 'DEBUG_P2P_DISABLE_COLOCATION_PENALTY'
   | 'DEPLOY_AZTEC_CONTRACTS_SALT'
@@ -173,7 +177,6 @@ export type EnvVar =
   | 'L1_READER_VIEM_POLLING_INTERVAL_MS'
   | 'PROVER_VIEM_POLLING_INTERVAL_MS'
   | 'SEQ_VIEM_POLLING_INTERVAL_MS'
-  | 'WS_DB_MAP_SIZE_KB'
   | 'WS_DATA_DIRECTORY'
   | 'WS_NUM_HISTORIC_BLOCKS'
   | 'ETHEREUM_SLOT_DURATION'

yarn-project/kv-store/src/config.ts

@@ -20,7 +20,9 @@ export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
   },
   l1Contracts: {
     description: 'The deployed L1 contract addresses',
-    nested: l1ContractAddressesMapping,
+    nested: {
+      rollupAddress: l1ContractAddressesMapping.rollupAddress,
+    },
   },
 };
 

yarn-project/p2p/src/client/factory.ts

@@ -36,7 +36,7 @@ export const createP2PClient = async <T extends P2PClientType>(
   telemetry: TelemetryClient = getTelemetryClient(),
   deps: P2PClientDeps<T> = {},
 ) => {
-  let config = { ..._config };
+  let config = { ..._config, dataStoreMapSizeKB: _config.p2pStoreMapSizeKb ?? _config.dataStoreMapSizeKB };
   const logger = deps.logger ?? createLogger('p2p');
   const store = deps.store ?? (await createStore('p2p', 1, config, createLogger('p2p:lmdb-v2')));
   const archive = await createStore('p2p-archive', 1, config, createLogger('p2p-archive:lmdb-v2'));

yarn-project/p2p/src/config.ts

@@ -164,6 +164,11 @@ export interface P2PConfig extends P2PReqRespConfig, ChainConfig {
    * A list of trusted peers.
    */
   trustedPeers: string[];
+
+  /**
+   * The maximum possible size of the P2P DB in KB. Overwrites the general dataStoreMapSizeKB.
+   */
+  p2pStoreMapSizeKb?: number;
 }
 
 export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
@@ -331,6 +336,11 @@ export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
     description: 'A list of trusted peers ENRs. Separated by commas.',
     defaultValue: [],
   },
+  p2pStoreMapSizeKb: {
+    env: 'P2P_STORE_MAP_SIZE_KB',
+    parseEnv: (val: string | undefined) => (val ? +val : undefined),
+    description: 'The maximum possible size of the P2P DB in KB. Overwrites the general dataStoreMapSizeKB.',
+  },
   ...p2pReqRespConfigMappings,
   ...chainConfigMappings,
 };

yarn-project/prover-client/src/proving_broker/config.ts

@@ -21,6 +21,8 @@ export const ProverBrokerConfig = z.object({
   dataDirectory: z.string().optional(),
   /** The size of the data store map */
   dataStoreMapSizeKB: z.number().int().nonnegative(),
+  /** The size of the prover broker's database. Will override the dataStoreMapSizeKB if set. */
+  proverBrokerStoreMapSizeKB: z.number().int().nonnegative().optional(),
   /** The prover broker may batch jobs together before writing to the database */
   proverBrokerBatchSize: z.number().int().nonnegative(),
   /** How often the job batches get flushed */
@@ -64,6 +66,11 @@ export const proverBrokerConfigMappings: ConfigMappingsType<ProverBrokerConfig>
     description: 'The maximum number of epochs to keep results for',
     ...numberConfigHelper(1),
   },
+  proverBrokerStoreMapSizeKB: {
+    env: 'PROVER_BROKER_STORE_MAP_SIZE_KB',
+    parseEnv: (val: string | undefined) => (val ? +val : undefined),
+    description: "The size of the prover broker's database. Will override the dataStoreMapSizeKB if set.",
+  },
   ...l1ReaderConfigMappings,
   ...dataConfigMappings,
 };

yarn-project/prover-client/src/proving_broker/factory.ts

@@ -6,9 +6,10 @@ import { InMemoryBrokerDatabase } from './proving_broker_database/memory.js';
 import { KVBrokerDatabase } from './proving_broker_database/persisted.js';
 
 export async function createAndStartProvingBroker(
-  config: ProverBrokerConfig,
+  _config: ProverBrokerConfig,
   client: TelemetryClient,
 ): Promise<ProvingBroker> {
+  const config = { ..._config, dataStoreMapSizeKB: _config.proverBrokerStoreMapSizeKB ?? _config.dataStoreMapSizeKB };
   const database = config.dataDirectory ? await KVBrokerDatabase.new(config, client) : new InMemoryBrokerDatabase();
 
   const broker = new ProvingBroker(database, config, client);

yarn-project/world-state/src/synchronizer/config.ts

@@ -46,7 +46,7 @@ export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
   worldStateDbMapSizeKb: {
     env: 'WS_DB_MAP_SIZE_KB',
     parseEnv: (val: string | undefined) => (val ? +val : undefined),
-    description: 'The maximum possible size of the world state DB',
+    description: 'The maximum possible size of the world state DB in KB. Overwrites the general dataStoreMapSizeKB.',
   },
   worldStateDataDirectory: {
     env: 'WS_DATA_DIRECTORY',