diff --git a/spartan/scripts/deploy_network.sh b/spartan/scripts/deploy_network.sh index 44ae37f8a9d8..c714f837c9a9 100755 --- a/spartan/scripts/deploy_network.sh +++ b/spartan/scripts/deploy_network.sh @@ -522,6 +522,8 @@ FISHERMAN_MNEMONIC_START_INDEX = ${FISHERMAN_MNEMONIC_START_INDEX} FULL_NODE_REPLICAS = ${FULL_NODE_REPLICAS:-1} PROVER_FAILED_PROOF_STORE = "${PROVER_FAILED_PROOF_STORE}" +PROVER_PROOF_STORE = "${PROVER_PROOF_STORE:-}" +PROVER_BROKER_DEBUG_REPLAY_ENABLED = ${PROVER_BROKER_DEBUG_REPLAY_ENABLED:-false} DEPLOY_ARCHIVAL_NODE = ${DEPLOY_ARCHIVAL_NODE} PROVER_REPLICAS = ${PROVER_REPLICAS} diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index b8f84067fd7d..564c2380ed03 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -311,6 +311,7 @@ locals { "node.node.proverRealProofs" = var.PROVER_REAL_PROOFS "node.node.logLevel" = var.LOG_LEVEL "node.node.env.PROVER_FAILED_PROOF_STORE" = var.PROVER_FAILED_PROOF_STORE + "node.node.env.PROVER_PROOF_STORE" = var.PROVER_PROOF_STORE "node.node.env.DEBUG_FORCE_TX_PROOF_VERIFICATION" = var.DEBUG_FORCE_TX_PROOF_VERIFICATION "node.node.env.KEY_INDEX_START" = var.PROVER_PUBLISHER_MNEMONIC_START_INDEX "node.node.env.PUBLISHER_KEY_INDEX_START" = var.PROVER_PUBLISHER_MNEMONIC_START_INDEX @@ -324,6 +325,7 @@ locals { "broker.node.proverRealProofs" = var.PROVER_REAL_PROOFS "broker.node.logLevel" = var.LOG_LEVEL "broker.node.env.BOOTSTRAP_NODES" = "asdf" + "broker.node.env.PROVER_BROKER_DEBUG_REPLAY_ENABLED" = var.PROVER_BROKER_DEBUG_REPLAY_ENABLED "agent.node.proverRealProofs" = var.PROVER_REAL_PROOFS "agent.node.env.PROVER_AGENT_POLL_INTERVAL_MS" = var.PROVER_AGENT_POLL_INTERVAL_MS "agent.replicaCount" = var.PROVER_REPLICAS @@ -331,6 +333,7 @@ locals { "agent.node.env.PROVER_AGENT_COUNT" = var.PROVER_AGENTS_PER_PROVER "agent.node.env.PROVER_TEST_DELAY_TYPE" = var.PROVER_TEST_DELAY_TYPE "agent.node.env.PROVER_AGENT_PROOF_TYPES" = join(",", var.PROVER_AGENT_PROOF_TYPES) + "agent.node.env.PROVER_PROOF_STORE" = var.PROVER_PROOF_STORE "agent.node.otelIncludeMetrics" = var.PROVER_AGENT_INCLUDE_METRICS "agent.node.logLevel" = var.LOG_LEVEL "node.node.env.L1_PRIORITY_FEE_BUMP_PERCENTAGE" = var.PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAGE diff --git a/spartan/terraform/deploy-aztec-infra/variables.tf b/spartan/terraform/deploy-aztec-infra/variables.tf index 6c507bbd1e90..c1145fdd93c9 100644 --- a/spartan/terraform/deploy-aztec-infra/variables.tf +++ b/spartan/terraform/deploy-aztec-infra/variables.tf @@ -576,6 +576,19 @@ variable "PROVER_FAILED_PROOF_STORE" { default = "" } +variable "PROVER_PROOF_STORE" { + description = "Optional GCS/S3/file URI to store proof inputs and outputs (e.g. gs://bucket/path, s3://bucket/path, file:///path)" + type = string + nullable = false + default = "" +} + +variable "PROVER_BROKER_DEBUG_REPLAY_ENABLED" { + description = "Enable debug replay mode for the prover broker to replay proving jobs from stored inputs" + type = bool + default = false +} + variable "RPC_REPLICAS" { description = "The number of RPC replicas" type = string diff --git a/yarn-project/aztec/src/cli/cmds/start_prover_agent.ts b/yarn-project/aztec/src/cli/cmds/start_prover_agent.ts index 47b210d08fa0..8189893e5948 100644 --- a/yarn-project/aztec/src/cli/cmds/start_prover_agent.ts +++ b/yarn-project/aztec/src/cli/cmds/start_prover_agent.ts @@ -4,9 +4,9 @@ import { Agent, makeUndiciFetch } from '@aztec/foundation/json-rpc/undici'; import type { LogFn } from '@aztec/foundation/log'; import { buildServerCircuitProver } from '@aztec/prover-client'; import { - InlineProofStore, type ProverAgentConfig, ProvingAgent, + createProofStore, createProvingJobBrokerClient, proverAgentConfigMappings, } from '@aztec/prover-client/broker'; @@ -55,7 +55,7 @@ export async function startProverAgent( const telemetry = await initTelemetryClient(extractRelevantOptions(options, telemetryClientConfigMappings, 'tel')); const prover = await buildServerCircuitProver(config, telemetry); - const proofStore = new InlineProofStore(); + const proofStore = await createProofStore(config.proofStore); const agents = times( config.proverAgentCount, () => new ProvingAgent(broker, proofStore, prover, config.proverAgentProofTypes, config.proverAgentPollIntervalMs), diff --git a/yarn-project/aztec/src/cli/cmds/start_prover_broker.ts b/yarn-project/aztec/src/cli/cmds/start_prover_broker.ts index 409318db4491..ae3d087b02dd 100644 --- a/yarn-project/aztec/src/cli/cmds/start_prover_broker.ts +++ b/yarn-project/aztec/src/cli/cmds/start_prover_broker.ts @@ -5,6 +5,7 @@ import type { LogFn } from '@aztec/foundation/log'; import { type ProverBrokerConfig, ProvingJobBrokerSchema, + ProvingJobBrokerSchemaWithDebug, createAndStartProvingBroker, proverBrokerConfigMappings, } from '@aztec/prover-client/broker'; @@ -59,7 +60,10 @@ export async function startProverBroker( ); } - services.proverBroker = [broker, ProvingJobBrokerSchema]; + services.proverBroker = [ + broker, + config.proverBrokerDebugReplayEnabled ? ProvingJobBrokerSchemaWithDebug : ProvingJobBrokerSchema, + ]; signalHandlers.push(() => broker.stop()); return { broker, config }; diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 7325ee0cd19e..2c89b8ee1e49 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -156,8 +156,10 @@ export type EnvVar = | 'PROVER_BROKER_BATCH_INTERVAL_MS' | 'PROVER_BROKER_BATCH_SIZE' | 'PROVER_BROKER_MAX_EPOCHS_TO_KEEP_RESULTS_FOR' + | 'PROVER_BROKER_DEBUG_REPLAY_ENABLED' | 'PROVER_CANCEL_JOBS_ON_STOP' | 'PROVER_COORDINATION_NODE_URLS' + | 'PROVER_PROOF_STORE' | 'PROVER_FAILED_PROOF_STORE' | 'PROVER_NODE_FAILED_EPOCH_STORE' | 'PROVER_NODE_DISABLE_PROOF_PUBLISH' diff --git a/yarn-project/prover-client/src/prover-client/prover-client.ts b/yarn-project/prover-client/src/prover-client/prover-client.ts index c8cd91afc8a3..89b866220c06 100644 --- a/yarn-project/prover-client/src/prover-client/prover-client.ts +++ b/yarn-project/prover-client/src/prover-client/prover-client.ts @@ -29,20 +29,16 @@ export class ProverClient implements EpochProverManager { private running = false; private agents: ProvingAgent[] = []; - private proofStore: ProofStore; - private failedProofStore: ProofStore | undefined; - private constructor( private config: ProverClientConfig, private worldState: ForkMerkleTreeOperations & ReadonlyWorldStateAccess, private orchestratorClient: ProvingJobProducer, + private proofStore: ProofStore, + private failedProofStore: ProofStore | undefined, private agentClient?: ProvingJobConsumer, private telemetry: TelemetryClient = getTelemetryClient(), private log: Logger = createLogger('prover-client:tx-prover'), - ) { - this.proofStore = new InlineProofStore(); - this.failedProofStore = this.config.failedProofStore ? createProofStore(this.config.failedProofStore) : undefined; - } + ) {} public createEpochProver(): EpochProver { const bindings = this.log.getBindings(); @@ -118,7 +114,9 @@ export class ProverClient implements EpochProverManager { broker: ProvingJobBroker, telemetry: TelemetryClient = getTelemetryClient(), ) { - const prover = new ProverClient(config, worldState, broker, broker, telemetry); + const proofStore = await createProofStore(config.proofStore); + const failedProofStore = config.failedProofStore ? await createProofStore(config.failedProofStore) : undefined; + const prover = new ProverClient(config, worldState, broker, proofStore, failedProofStore, broker, telemetry); await prover.start(); return prover; } diff --git a/yarn-project/prover-client/src/proving_broker/config.ts b/yarn-project/prover-client/src/proving_broker/config.ts index eff4f73f262c..d8b875858ad8 100644 --- a/yarn-project/prover-client/src/proving_broker/config.ts +++ b/yarn-project/prover-client/src/proving_broker/config.ts @@ -31,6 +31,8 @@ export const ProverBrokerConfig = z.object({ proverBrokerBatchIntervalMs: z.number().int().nonnegative(), /** The maximum number of epochs to keep results for */ proverBrokerMaxEpochsToKeepResultsFor: z.number().int().nonnegative(), + /** Enable debug replay mode for replaying proving jobs from stored inputs */ + proverBrokerDebugReplayEnabled: z.boolean(), }); export type ProverBrokerConfig = z.infer & @@ -74,6 +76,11 @@ export const proverBrokerConfigMappings: ConfigMappingsType parseEnv: (val: string | undefined) => (val ? +val : undefined), description: "The size of the prover broker's database. Will override the dataStoreMapSizeKb if set.", }, + proverBrokerDebugReplayEnabled: { + env: 'PROVER_BROKER_DEBUG_REPLAY_ENABLED', + description: 'Enable debug replay mode for replaying proving jobs from stored inputs', + ...booleanConfigHelper(false), + }, ...dataConfigMappings, ...l1ReaderConfigMappings, ...pickConfigMappings(chainConfigMappings, ['rollupVersion']), @@ -102,6 +109,8 @@ export const ProverAgentConfig = z.object({ proverTestVerificationDelayMs: z.number().optional(), /** Whether to abort pending proving jobs when the orchestrator is cancelled */ cancelJobsOnStop: z.boolean(), + /** Where to store proving results. Must be accessible to both prover node and agents. If not set will inline-encode the parameters */ + proofStore: z.string().optional(), }); export type ProverAgentConfig = z.infer; @@ -162,4 +171,8 @@ export const proverAgentConfigMappings: ConfigMappingsType = 'When false (default), jobs remain in the broker queue and can be reused on restart/reorg.', ...booleanConfigHelper(false), }, + proofStore: { + env: 'PROVER_PROOF_STORE', + description: 'Optional proof input store for the prover', + }, }; diff --git a/yarn-project/prover-client/src/proving_broker/proof_store/factory.ts b/yarn-project/prover-client/src/proving_broker/proof_store/factory.ts index 3abbc573de2f..8f1af78b9c0c 100644 --- a/yarn-project/prover-client/src/proving_broker/proof_store/factory.ts +++ b/yarn-project/prover-client/src/proving_broker/proof_store/factory.ts @@ -1,42 +1,20 @@ import { createLogger } from '@aztec/foundation/log'; +import { createFileStore } from '@aztec/stdlib/file-store'; -import { GoogleCloudStorageProofStore } from './gcs_proof_store.js'; +import { FileStoreProofStore } from './file_store_proof_store.js'; import { InlineProofStore } from './inline_proof_store.js'; import type { ProofStore } from './proof_store.js'; -export function createProofStore(config: string | undefined, logger = createLogger('prover-client:proof-store')) { - if (config === undefined) { +export async function createProofStore( + config: string | undefined, + logger = createLogger('prover-client:proof-store'), +): Promise { + if (!config) { logger.info('Creating inline proof store'); return new InlineProofStore(); - } else if (config.startsWith('gs://')) { - try { - const url = new URL(config); - const bucket = url.host; - const path = url.pathname.replace(/^\/+/, ''); - logger.info(`Creating google cloud proof store at ${bucket}`, { bucket, path }); - return new GoogleCloudStorageProofStore(bucket, path); - } catch { - throw new Error( - `Invalid google cloud proof store definition: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`, - ); - } - } else { - throw new Error(`Unknown proof store config: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`); } -} -export function createProofStoreForUri( - uri: string, - logger = createLogger('prover-client:proof-store'), -): Pick { - if (uri.startsWith('data://')) { - return createProofStore(undefined, logger); - } else if (uri.startsWith('gs://')) { - const url = new URL(uri); - const basePath = url.pathname.replace(/^\/+/, '').split('/').slice(0, -3); - url.pathname = basePath.join('/'); - return createProofStore(uri, logger); - } else { - throw new Error(`Unknown proof store config: '${uri}'. Supported protocols are 'data://' and 'gs://'.`); - } + const fileStore = await createFileStore(config, logger); + logger.info(`Creating file store proof store at ${config}`); + return new FileStoreProofStore(fileStore); } diff --git a/yarn-project/prover-client/src/proving_broker/proof_store/file_store_proof_store.ts b/yarn-project/prover-client/src/proving_broker/proof_store/file_store_proof_store.ts new file mode 100644 index 000000000000..49e8d0402d65 --- /dev/null +++ b/yarn-project/prover-client/src/proving_broker/proof_store/file_store_proof_store.ts @@ -0,0 +1,78 @@ +import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc'; +import type { FileStore } from '@aztec/stdlib/file-store'; +import { + type ProofUri, + type ProvingJobId, + type ProvingJobInputs, + type ProvingJobInputsMap, + ProvingJobResult, + type ProvingJobResultsMap, + getProvingJobInputClassFor, +} from '@aztec/stdlib/interfaces/server'; +import { ProvingRequestType } from '@aztec/stdlib/proofs'; + +import type { ProofStore } from './proof_store.js'; + +const INPUTS_PATH = 'inputs'; +const OUTPUTS_PATH = 'outputs'; + +/** + * A proof store implementation backed by a generic FileStore. + * Supports any storage backend (GCS, S3, local filesystem) via the FileStore abstraction. + */ +export class FileStoreProofStore implements ProofStore { + constructor(private readonly fileStore: FileStore) {} + + async saveProofInput( + id: ProvingJobId, + type: T, + inputs: ProvingJobInputsMap[T], + ): Promise { + const path = `${INPUTS_PATH}/${ProvingRequestType[type]}/${id}`; + const uri = await this.fileStore.save(path, inputs.toBuffer()); + return uri as ProofUri; + } + + async saveProofOutput( + id: ProvingJobId, + type: T, + result: ProvingJobResultsMap[T], + ): Promise { + const jobResult = { type, result } as ProvingJobResult; + const json = jsonStringify(jobResult); + const path = `${OUTPUTS_PATH}/${ProvingRequestType[type]}/${id}.json`; + const uri = await this.fileStore.save(path, Buffer.from(json, 'utf-8')); + return uri as ProofUri; + } + + async getProofInput(uri: ProofUri): Promise { + try { + const buffer = await this.fileStore.read(uri); + const type = this.extractTypeFromUri(uri); + const inputs = getProvingJobInputClassFor(type).fromBuffer(buffer); + return { inputs, type } as ProvingJobInputs; + } catch (err) { + throw new Error(`Error getting proof input at ${uri}: ${err}`); + } + } + + async getProofOutput(uri: ProofUri): Promise { + try { + const buffer = await this.fileStore.read(uri); + return jsonParseWithSchema(buffer.toString('utf-8'), ProvingJobResult); + } catch (err) { + throw new Error(`Error getting proof output at ${uri}: ${err}`); + } + } + + private extractTypeFromUri(uri: string): ProvingRequestType { + const url = new URL(uri); + const pathParts = url.pathname.split('/').filter(Boolean); + const typeString = pathParts.at(-2); + const type = typeString ? ProvingRequestType[typeString as keyof typeof ProvingRequestType] : undefined; + if (type === undefined) { + throw new Error(`Unrecognized proof type ${typeString} in URI ${uri}`); + } + return type; + } +} diff --git a/yarn-project/prover-client/src/proving_broker/proof_store/gcs_proof_store.ts b/yarn-project/prover-client/src/proving_broker/proof_store/gcs_proof_store.ts deleted file mode 100644 index 8ee35ea89c06..000000000000 --- a/yarn-project/prover-client/src/proving_broker/proof_store/gcs_proof_store.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { - type ProofUri, - type ProvingJobId, - type ProvingJobInputs, - type ProvingJobInputsMap, - type ProvingJobResult, - type ProvingJobResultsMap, - getProvingJobInputClassFor, -} from '@aztec/stdlib/interfaces/server'; -import { ProvingRequestType } from '@aztec/stdlib/proofs'; - -import { Storage } from '@google-cloud/storage'; -import { join } from 'path'; - -import type { ProofStore } from './proof_store.js'; - -const INPUTS_PATH = 'inputs'; - -// REFACTOR(#13067): Use the stdlib/file-store instead of referencing google-cloud-storage directly. -export class GoogleCloudStorageProofStore implements ProofStore { - private readonly storage: Storage; - - constructor( - private readonly bucketName: string, - private readonly path: string, - ) { - this.storage = new Storage(); - } - - public async saveProofInput( - id: ProvingJobId, - type: T, - inputs: ProvingJobInputsMap[T], - ): Promise { - const path = join(this.path, INPUTS_PATH, ProvingRequestType[type], id); - const file = this.storage.bucket(this.bucketName).file(path); - await file.save(inputs.toBuffer()); - return file.cloudStorageURI.toString() as ProofUri; - } - - saveProofOutput( - _id: ProvingJobId, - _type: T, - _result: ProvingJobResultsMap[T], - ): Promise { - throw new Error('Not implemented'); - } - - public async getProofInput(uri: ProofUri): Promise { - try { - const url = new URL(uri); - const bucket = this.storage.bucket(url.host); - const path = url.pathname.replace(/^\/+/, ''); - const file = bucket.file(path); - if (!(await file.exists())) { - throw new Error(`File at ${uri} does not exist`); - } - - const typeString = path.split('/').at(-2); - const type = typeString ? ProvingRequestType[typeString as keyof typeof ProvingRequestType] : undefined; - if (type === undefined) { - throw new Error(`Unrecognized proof type ${type} in path ${path}`); - } - - const contents = await file.download(); - const inputs = getProvingJobInputClassFor(type).fromBuffer(contents[0]); - return { inputs, type } as ProvingJobInputs; - } catch (err) { - throw new Error(`Error getting proof input at ${uri}: ${err}`); - } - } - - getProofOutput(_uri: ProofUri): Promise { - throw new Error('Not implemented'); - } -} diff --git a/yarn-project/prover-client/src/proving_broker/proof_store/index.ts b/yarn-project/prover-client/src/proving_broker/proof_store/index.ts index 37896541cd41..9a24dca30674 100644 --- a/yarn-project/prover-client/src/proving_broker/proof_store/index.ts +++ b/yarn-project/prover-client/src/proving_broker/proof_store/index.ts @@ -1,4 +1,4 @@ export * from './proof_store.js'; export * from './inline_proof_store.js'; export * from './factory.js'; -export * from './gcs_proof_store.js'; +export * from './file_store_proof_store.js'; diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts b/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts index e0b6d078112c..fd461afbd2ce 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts @@ -60,6 +60,7 @@ describe.each([ proverBrokerPollIntervalMs: brokerIntervalMs, proverBrokerJobMaxRetries: maxRetries, proverBrokerMaxEpochsToKeepResultsFor: 1, + proverBrokerDebugReplayEnabled: false, }); }); diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker.ts b/yarn-project/prover-client/src/proving_broker/proving_broker.ts index 094cb57be4a9..e9d7c191e977 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker.ts @@ -7,6 +7,7 @@ import { type GetProvingJobResponse, type ProofUri, type ProvingJob, + type ProvingJobBrokerDebug, type ProvingJobConsumer, type ProvingJobFilter, type ProvingJobId, @@ -36,7 +37,7 @@ type EnqueuedProvingJob = Pick; * A broker that manages proof requests and distributes them to workers based on their priority. * It takes a backend that is responsible for storing and retrieving proof requests and results. */ -export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable { +export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, ProvingJobBrokerDebug, Traceable { private queues: ProvingQueues = { [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue(provingJobComparator), [ProvingRequestType.PUBLIC_CHONK_VERIFIER]: new PriorityMemoryQueue(provingJobComparator), @@ -114,6 +115,8 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr private started = false; + private debugReplayEnabled: boolean; + public constructor( private database: ProvingBrokerDatabase, { @@ -121,6 +124,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor, + proverBrokerDebugReplayEnabled, }: Required< Pick< ProverBrokerConfig, @@ -128,6 +132,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr | 'proverBrokerPollIntervalMs' | 'proverBrokerJobMaxRetries' | 'proverBrokerMaxEpochsToKeepResultsFor' + | 'proverBrokerDebugReplayEnabled' > > = defaultProverBrokerConfig, client: TelemetryClient = getTelemetryClient(), @@ -139,6 +144,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr this.jobTimeoutMs = proverBrokerJobTimeoutMs!; this.maxRetries = proverBrokerJobMaxRetries!; this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor!; + this.debugReplayEnabled = proverBrokerDebugReplayEnabled ?? false; } private measureQueueDepth: MonitorCallback = (type: ProvingRequestType) => { @@ -241,6 +247,29 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter)); } + public async replayProvingJob( + jobId: ProvingJobId, + type: ProvingRequestType, + epochNumber: EpochNumber, + inputsUri: ProofUri, + ): Promise { + if (!this.debugReplayEnabled) { + throw new Error('Debug replay not enabled. Set PROVER_BROKER_DEBUG_REPLAY_ENABLED=true'); + } + + this.logger.info(`Replaying proving job`, { provingJobId: jobId, epochNumber, inputsUri }); + + // Clear existing state and enqueue + this.cleanUpProvingJobState([jobId]); + + const job: ProvingJob = { id: jobId, type, epochNumber, inputsUri }; + this.jobsCache.set(jobId, job); + await this.database.addProvingJob(job); + this.enqueueJobInternal(job); + + return { status: 'in-queue' }; + } + async #enqueueProvingJob(job: ProvingJob): Promise { // We return the job status at the start of this call const jobStatus = this.#getProvingJobStatus(job.id); diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker_agent_integration.test.ts b/yarn-project/prover-client/src/proving_broker/proving_broker_agent_integration.test.ts index 655d62952d05..e52e3791c04c 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker_agent_integration.test.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker_agent_integration.test.ts @@ -35,6 +35,7 @@ describe('ProvingBroker <-> ProvingAgent integration', () => { proverBrokerJobMaxRetries: 3, proverBrokerPollIntervalMs: WORK_LOOP, proverBrokerMaxEpochsToKeepResultsFor: 1, + proverBrokerDebugReplayEnabled: false, }); addBrokerDelay('getProvingJob', 5, 50); diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker_agent_rpc_integration.test.ts b/yarn-project/prover-client/src/proving_broker/proving_broker_agent_rpc_integration.test.ts index 67141b99a283..00b2ef014fd9 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker_agent_rpc_integration.test.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker_agent_rpc_integration.test.ts @@ -17,6 +17,7 @@ describe('ProvingBroker RPC', () => { proverBrokerPollIntervalMs: 100, proverBrokerJobMaxRetries: 3, proverBrokerMaxEpochsToKeepResultsFor: 1, + proverBrokerDebugReplayEnabled: false, }); await broker.start(); diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker_database/broker_persisted_database.test.ts b/yarn-project/prover-client/src/proving_broker/proving_broker_database/broker_persisted_database.test.ts index 470189cca7df..8eb3ecddfcc1 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker_database/broker_persisted_database.test.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker_database/broker_persisted_database.test.ts @@ -30,6 +30,7 @@ describe('ProvingBrokerPersistedDatabase', () => { proverBrokerBatchSize: 1, proverBrokerBatchIntervalMs: 10, proverBrokerMaxEpochsToKeepResultsFor: 1, + proverBrokerDebugReplayEnabled: false, l1Contracts: { rollupAddress: EthAddress.random(), } as any, @@ -339,6 +340,7 @@ describe('ProvingBrokerPersistedDatabase', () => { proverBrokerBatchSize: batchSize, proverBrokerBatchIntervalMs: 10, proverBrokerMaxEpochsToKeepResultsFor: 1, + proverBrokerDebugReplayEnabled: false, l1Contracts: { rollupAddress: EthAddress.random(), } as any, diff --git a/yarn-project/prover-client/src/proving_broker/rpc.ts b/yarn-project/prover-client/src/proving_broker/rpc.ts index 83f2a025e4a8..4d0d94c4869e 100644 --- a/yarn-project/prover-client/src/proving_broker/rpc.ts +++ b/yarn-project/prover-client/src/proving_broker/rpc.ts @@ -1,9 +1,11 @@ +import { EpochNumberSchema } from '@aztec/foundation/branded-types'; import { createSafeJsonRpcClient } from '@aztec/foundation/json-rpc/client'; import { type GetProvingJobResponse, ProofUri, ProvingJob, type ProvingJobBroker, + type ProvingJobBrokerDebug, type ProvingJobConsumer, ProvingJobId, type ProvingJobProducer, @@ -53,6 +55,18 @@ export const ProvingJobBrokerSchema: ApiSchemaFor = { ...ProvingJobProducerSchema, }; +export const ProvingJobBrokerDebugSchema: ApiSchemaFor = { + replayProvingJob: z + .function() + .args(ProvingJobId, z.nativeEnum(ProvingRequestType), EpochNumberSchema, ProofUri) + .returns(ProvingJobStatus), +}; + +export const ProvingJobBrokerSchemaWithDebug: ApiSchemaFor = { + ...ProvingJobBrokerSchema, + ...ProvingJobBrokerDebugSchema, +}; + export function createProvingJobBrokerClient( url: string, versions: Partial, diff --git a/yarn-project/stdlib/src/interfaces/prover-broker.ts b/yarn-project/stdlib/src/interfaces/prover-broker.ts index aa543b1c064c..1dc89e852fec 100644 --- a/yarn-project/stdlib/src/interfaces/prover-broker.ts +++ b/yarn-project/stdlib/src/interfaces/prover-broker.ts @@ -1,3 +1,5 @@ +import type { EpochNumber } from '@aztec/foundation/branded-types'; + import type { ProvingRequestType } from '../proofs/proving_request_type.js'; import type { ProofUri, ProvingJob, ProvingJobId, ProvingJobStatus } from './proving-job.js'; @@ -88,3 +90,23 @@ export interface ProvingJobConsumer { } export interface ProvingJobBroker extends ProvingJobProducer, ProvingJobConsumer {} + +/** + * Debug interface for replaying proving jobs from stored inputs. + * Used for benchmarking different agent configurations against the same workload. + */ +export interface ProvingJobBrokerDebug { + /** + * Replays a proving job by re-enqueuing it with inputs from the configured proof store. + * The proof type is parsed from the job ID (format: epoch:typeName:hash). + * @param jobId - The original job ID to replay + * @param epochNumber - The epoch number to assign + * @param inputsUri - The proof inputs location + */ + replayProvingJob( + jobId: ProvingJobId, + type: ProvingRequestType, + epochNumber: EpochNumber, + inputsUri: ProofUri, + ): Promise; +} diff --git a/yarn-project/stdlib/src/interfaces/prover-client.ts b/yarn-project/stdlib/src/interfaces/prover-client.ts index 9d1e41ac00e9..1c6f44df7690 100644 --- a/yarn-project/stdlib/src/interfaces/prover-client.ts +++ b/yarn-project/stdlib/src/interfaces/prover-client.ts @@ -35,6 +35,8 @@ export type ProverConfig = ActualProverConfig & { proverId?: EthAddress; /** Number of proving agents to start within the prover. */ proverAgentCount: number; + /** Where to store proving request. Must be accessible to both prover node and agents. If not set will inline-encode the parameters */ + proofStore?: string; /** Store for failed proof inputs. */ failedProofStore?: string; }; @@ -48,6 +50,7 @@ export const ProverConfigSchema = zodFor()( proverTestDelayMs: z.number(), proverTestDelayFactor: z.number(), proverAgentCount: z.number(), + proofStore: z.string().optional(), failedProofStore: z.string().optional(), cancelJobsOnStop: z.boolean(), }), @@ -87,6 +90,10 @@ export const proverConfigMappings: ConfigMappingsType = { description: 'The number of prover agents to start', ...numberConfigHelper(1), }, + proofStore: { + env: 'PROVER_PROOF_STORE', + description: 'Optional proof input store for the prover', + }, failedProofStore: { env: 'PROVER_FAILED_PROOF_STORE', description: