feat: first pass

yarn-project/archiver/src/test/mock_l2_block_source.ts

@@ -17,7 +17,12 @@ import {
 } from '@aztec/stdlib/block';
 import { Checkpoint, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
 import type { ContractClassPublic, ContractDataSource, ContractInstanceWithAddress } from '@aztec/stdlib/contract';
-import { EmptyL1RollupConstants, type L1RollupConstants, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
+import {
+  EmptyL1RollupConstants,
+  type L1RollupConstants,
+  getEpochAtSlot,
+  getSlotRangeForEpoch,
+} from '@aztec/stdlib/epoch-helpers';
 import { type BlockHeader, TxExecutionResult, TxHash, TxReceipt, TxStatus } from '@aztec/stdlib/tx';
 import type { UInt64 } from '@aztec/stdlib/types';
 
@@ -30,6 +35,7 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource {
   private provenBlockNumber: number = 0;
   private finalizedBlockNumber: number = 0;
   private checkpointedBlockNumber: number = 0;
+  private currentSlotNumber: SlotNumber = SlotNumber(0);
 
   private log = createLogger('archiver:mock_l2_block_source');
 
@@ -40,6 +46,7 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource {
       this.l2Blocks.push(block);
     }
 
+    this.currentSlotNumber = SlotNumber(this.l2Blocks[this.l2Blocks.length - 1].header.globalVariables.slotNumber);
     this.log.verbose(`Created ${numBlocks} blocks in the mock L2 block source`);
   }
 
@@ -68,6 +75,10 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource {
     this.checkpointedBlockNumber = checkpointedBlockNumber;
   }
 
+  public setCurrentSlotNumber(slot: SlotNumber) {
+    this.currentSlotNumber = slot;
+  }
+
   /**
    * Method to fetch the rollup contract address at the base-layer.
    * @returns The rollup address.
@@ -396,11 +407,12 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource {
   }
 
   getL2EpochNumber(): Promise<EpochNumber> {
-    throw new Error('Method not implemented.');
+    const epochDuration = DefaultL1ContractsConfig.aztecEpochDuration;
+    return Promise.resolve(getEpochAtSlot(this.currentSlotNumber, { epochDuration }));
   }
 
   getL2SlotNumber(): Promise<SlotNumber> {
-    throw new Error('Method not implemented.');
+    return Promise.resolve(this.currentSlotNumber);
   }
 
   isEpochComplete(_epochNumber: EpochNumber): Promise<boolean> {

yarn-project/end-to-end/src/fixtures/e2e_prover_test.ts

@@ -255,6 +255,7 @@ export class FullProverTest {
       txGatheringTimeoutMs: 24_000,
       proverNodeFailedEpochStore: undefined,
       proverNodeEpochProvingDelayMs: undefined,
+      proverNodeOptimisticProcessing: true,
     };
     const sponsoredFPCAddress = await getSponsoredFPCAddress();
     const { prefilledPublicData } = await getGenesisValues(

yarn-project/end-to-end/src/fixtures/setup.ts

@@ -739,6 +739,7 @@ export function createAndSyncProverNode(
       proverNodeFailedEpochStore: undefined,
       proverId: EthAddress.fromNumber(1),
       proverNodeEpochProvingDelayMs: undefined,
+      proverNodeOptimisticProcessing: true,
       ...proverNodeConfig,
     };
 

yarn-project/foundation/src/config/env_var.ts

@@ -171,6 +171,7 @@ export type EnvVar =
   | 'PROVER_NODE_TX_GATHERING_BATCH_SIZE'
   | 'PROVER_NODE_TX_GATHERING_MAX_PARALLEL_REQUESTS_PER_NODE'
   | 'PROVER_NODE_TX_GATHERING_TIMEOUT_MS'
+  | 'PROVER_NODE_OPTIMISTIC_PROCESSING'
   | 'PROVER_PUBLISHER_PRIVATE_KEY'
   | 'PROVER_PUBLISHER_PRIVATE_KEYS'
   | 'PROVER_PUBLISHER_ADDRESSES'

yarn-project/p2p/src/client/p2p_client.ts

@@ -203,6 +203,8 @@ export class P2PClient<T extends P2PClientType = P2PClientType.Full>
         break;
       case 'chain-checkpointed':
         break;
+      case 'epoch-completed':
+        break;
       default: {
         const _: never = event;
         break;

yarn-project/prover-client/src/orchestrator/checkpoint-proving-state.ts

@@ -68,7 +68,7 @@ export class CheckpointProvingState {
     public readonly index: number,
     public readonly constants: CheckpointConstantData,
     public readonly totalNumBlocks: number,
-    private readonly finalBlobBatchingChallenges: FinalBlobBatchingChallenges,
+    private finalBlobBatchingChallenges: FinalBlobBatchingChallenges | undefined,
     private readonly headerOfLastBlockInPreviousCheckpoint: BlockHeader,
     private readonly lastArchiveSiblingPath: Tuple<Fr, typeof ARCHIVE_HEIGHT>,
     private readonly l1ToL2Messages: Fr[],
@@ -91,6 +91,11 @@ export class CheckpointProvingState {
     this.firstBlockNumber = BlockNumber(headerOfLastBlockInPreviousCheckpoint.globalVariables.blockNumber + 1);
   }
 
+  /** Sets the final blob batching challenges. Called when epoch structure is known. */
+  public setFinalBlobBatchingChallenges(challenges: FinalBlobBatchingChallenges) {
+    this.finalBlobBatchingChallenges = challenges;
+  }
+
   public get epochNumber(): number {
     return this.parentEpoch.epochNumber;
   }
@@ -283,6 +288,9 @@ export class CheckpointProvingState {
     if (!this.startBlobAccumulator) {
       throw new Error('Start blob accumulator is not set.');
     }
+    if (!this.finalBlobBatchingChallenges) {
+      throw new Error('Final blob batching challenges are not set.');
+    }
 
     // `blobFields` must've been set if `startBlobAccumulator` is set (in `accumulateBlobs`).
     const blobFields = this.blobFields!;

yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts

@@ -51,19 +51,25 @@ export type ProvingResult = { status: 'success' } | { status: 'failure'; reason:
  * Captures resolve and reject callbacks to provide a promise base interface to the consumer of our proving.
  */
 export class EpochProvingState {
-  private checkpointProofs: UnbalancedTreeStore<
-    ProofState<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
-  >;
+  // Deferred until setStructure() is called.
+  private checkpointProofs:
+    | UnbalancedTreeStore<ProofState<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>
+    | undefined;
   private checkpointPaddingProof:
     | ProofState<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
     | undefined;
   private rootRollupProof: ProofState<RootRollupPublicInputs, typeof NESTED_RECURSIVE_PROOF_LENGTH> | undefined;
   private checkpoints: (CheckpointProvingState | undefined)[] = [];
-  private startBlobAccumulator: BatchedBlobAccumulator;
+  // Deferred until setStructure() is called.
+  private startBlobAccumulator: BatchedBlobAccumulator | undefined;
   private endBlobAccumulator: BatchedBlobAccumulator | undefined;
   private finalBatchedBlob: BatchedBlob | undefined;
   private provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED;
 
+  // Deferred until setStructure() is called.
+  private _totalNumCheckpoints: number | undefined;
+  private _finalBlobBatchingChallenges: FinalBlobBatchingChallenges | undefined;
+
   // Map from tx hash to chonk verifier proof promise. Used when kickstarting chonk verifier proofs before tx processing.
   public readonly cachedChonkVerifierProofs = new Map<
     string,
@@ -74,12 +80,28 @@ export class EpochProvingState {
 
   constructor(
     public readonly epochNumber: EpochNumber,
-    public readonly totalNumCheckpoints: number,
-    private readonly finalBlobBatchingChallenges: FinalBlobBatchingChallenges,
     private onCheckpointBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => void,
     private completionCallback: (result: ProvingResult) => void,
     private rejectionCallback: (reason: string) => void,
-  ) {
+  ) {}
+
+  /** Returns the total number of checkpoints. Throws if structure has not been set. */
+  public get totalNumCheckpoints(): number {
+    if (this._totalNumCheckpoints === undefined) {
+      throw new Error('Epoch structure not set. Call setStructure() first.');
+    }
+    return this._totalNumCheckpoints;
+  }
+
+  /** Returns true if the epoch structure (totalNumCheckpoints, finalBlobBatchingChallenges) has been set. */
+  public hasStructure(): boolean {
+    return this._totalNumCheckpoints !== undefined;
+  }
+
+  /** Sets the epoch structure. Called when the epoch is complete. */
+  public setStructure(totalNumCheckpoints: number, finalBlobBatchingChallenges: FinalBlobBatchingChallenges) {
+    this._totalNumCheckpoints = totalNumCheckpoints;
+    this._finalBlobBatchingChallenges = finalBlobBatchingChallenges;
     this.checkpointProofs = new UnbalancedTreeStore(totalNumCheckpoints);
     this.startBlobAccumulator = BatchedBlobAccumulator.newWithChallenges(finalBlobBatchingChallenges);
   }
@@ -98,7 +120,8 @@ export class EpochProvingState {
     newL1ToL2MessageTreeSnapshot: AppendOnlyTreeSnapshot,
     newL1ToL2MessageSubtreeRootSiblingPath: Tuple<Fr, typeof L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH>,
   ): CheckpointProvingState {
-    if (checkpointIndex >= this.totalNumCheckpoints) {
+    // If structure is already set, validate the checkpoint index.
+    if (this.hasStructure() && checkpointIndex >= this.totalNumCheckpoints) {
       throw new Error(
         `Unable to start a new checkpoint at index ${checkpointIndex}. Expected at most ${this.totalNumCheckpoints} checkpoints.`,
       );
@@ -108,7 +131,7 @@ export class EpochProvingState {
       checkpointIndex,
       constants,
       totalNumBlocks,
-      this.finalBlobBatchingChallenges,
+      this._finalBlobBatchingChallenges,
       previousBlockHeader,
       lastArchiveSiblingPath,
       l1ToL2Messages,
@@ -121,7 +144,7 @@ export class EpochProvingState {
     );
     this.checkpoints[checkpointIndex] = checkpoint;
 
-    if (this.checkpoints.filter(c => !!c).length === this.totalNumCheckpoints) {
+    if (this.hasStructure() && this.checkpoints.filter(c => !!c).length === this.totalNumCheckpoints) {
       this.provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL;
     }
 
@@ -155,6 +178,10 @@ export class EpochProvingState {
 
   // Returns true if we are still able to accept checkpoints, false otherwise.
   public isAcceptingCheckpoints() {
+    // If structure isn't set yet, always accept checkpoints (we don't know the limit).
+    if (!this.hasStructure()) {
+      return true;
+    }
     return this.checkpoints.filter(c => !!c).length < this.totalNumCheckpoints;
   }
 
@@ -165,14 +192,14 @@ export class EpochProvingState {
       typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH
     >,
   ): TreeNodeLocation {
-    return this.checkpointProofs.setLeaf(checkpointIndex, { provingOutput });
+    return this.getCheckpointProofs().setLeaf(checkpointIndex, { provingOutput });
   }
 
   public tryStartProvingCheckpointMerge(location: TreeNodeLocation) {
-    if (this.checkpointProofs.getNode(location)?.isProving) {
+    if (this.getCheckpointProofs().getNode(location)?.isProving) {
       return false;
     } else {
-      this.checkpointProofs.setNode(location, { isProving: true });
+      this.getCheckpointProofs().setNode(location, { isProving: true });
       return true;
     }
   }
@@ -184,7 +211,7 @@ export class EpochProvingState {
       typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH
     >,
   ) {
-    this.checkpointProofs.setNode(location, { provingOutput });
+    this.getCheckpointProofs().setNode(location, { provingOutput });
   }
 
   public tryStartProvingRootRollup() {
@@ -219,6 +246,11 @@ export class EpochProvingState {
   }
 
   public async accumulateCheckpointOutHashes() {
+    // Cannot accumulate until structure is known.
+    if (!this.hasStructure()) {
+      return;
+    }
+
     const treeCalculator = await MerkleTreeCalculator.create(OUT_HASH_TREE_HEIGHT, undefined, (left, right) =>
       Promise.resolve(shaMerkleHash(left, right)),
     );
@@ -261,6 +293,11 @@ export class EpochProvingState {
   }
 
   public async setBlobAccumulators() {
+    // Cannot accumulate until structure is known and start blob accumulator is created.
+    if (!this.hasStructure() || !this.startBlobAccumulator) {
+      return;
+    }
+
     let previousAccumulator = this.startBlobAccumulator;
     // Accumulate blobs as far as we can for this epoch.
     for (let i = 0; i < this.totalNumCheckpoints; i++) {
@@ -292,11 +329,13 @@ export class EpochProvingState {
   }
 
   public getParentLocation(location: TreeNodeLocation) {
-    return this.checkpointProofs.getParentLocation(location);
+    return this.getCheckpointProofs().getParentLocation(location);
   }
 
   public getCheckpointMergeRollupInputs(mergeLocation: TreeNodeLocation) {
-    const [left, right] = this.checkpointProofs.getChildren(mergeLocation).map(c => c?.provingOutput);
+    const [left, right] = this.getCheckpointProofs()
+      .getChildren(mergeLocation)
+      .map(c => c?.provingOutput);
     if (!left || !right) {
       throw new Error('At least one child is not ready for the checkpoint merge rollup.');
     }
@@ -334,6 +373,9 @@ export class EpochProvingState {
   }
 
   public isReadyForCheckpointMerge(location: TreeNodeLocation) {
+    if (!this.checkpointProofs) {
+      return false;
+    }
     return !!this.checkpointProofs.getSibling(location)?.provingOutput;
   }
 
@@ -368,11 +410,20 @@ export class EpochProvingState {
     this.completionCallback(result);
   }
 
+  /** Returns the checkpointProofs store, asserting that structure has been set. */
+  private getCheckpointProofs() {
+    if (!this.checkpointProofs) {
+      throw new Error('Epoch structure not set. Call setStructure() first.');
+    }
+    return this.checkpointProofs;
+  }
+
   #getChildProofsForRoot() {
     const rootLocation = { level: 0, index: 0 };
+    const proofs = this.getCheckpointProofs();
     // If there's only 1 block, its block root proof will be stored at the root.
     return this.totalNumCheckpoints === 1
-      ? [this.checkpointProofs.getNode(rootLocation)?.provingOutput, this.checkpointPaddingProof?.provingOutput]
-      : this.checkpointProofs.getChildren(rootLocation).map(c => c?.provingOutput);
+      ? [proofs.getNode(rootLocation)?.provingOutput, this.checkpointPaddingProof?.provingOutput]
+      : proofs.getChildren(rootLocation).map(c => c?.provingOutput);
   }
 }

yarn-project/prover-client/src/orchestrator/orchestrator.ts

@@ -129,11 +129,7 @@ export class ProvingOrchestrator implements EpochProver {
     return Promise.resolve();
   }
 
-  public startNewEpoch(
-    epochNumber: EpochNumber,
-    totalNumCheckpoints: number,
-    finalBlobBatchingChallenges: FinalBlobBatchingChallenges,
-  ) {
+  public startNewEpoch(epochNumber: EpochNumber) {
     if (this.provingState?.verifyState()) {
       throw new Error(
         `Cannot start epoch ${epochNumber} when epoch ${this.provingState.epochNumber} is still being processed.`,
@@ -142,18 +138,42 @@ export class ProvingOrchestrator implements EpochProver {
 
     const { promise: _promise, resolve, reject } = promiseWithResolvers<ProvingResult>();
     const promise = _promise.catch((reason): ProvingResult => ({ status: 'failure', reason }));
-    this.logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`);
+    this.logger.info(`Starting epoch ${epochNumber}. Block-level proving can begin immediately.`);
     this.provingState = new EpochProvingState(
       epochNumber,
-      totalNumCheckpoints,
-      finalBlobBatchingChallenges,
       provingState => this.checkAndEnqueueCheckpointRootRollup(provingState),
       resolve,
       reject,
     );
     this.provingPromise = promise;
   }
 
+  public async setEpochStructure(
+    totalNumCheckpoints: number,
+    finalBlobBatchingChallenges: FinalBlobBatchingChallenges,
+  ) {
+    if (!this.provingState) {
+      throw new Error('Empty epoch proving state. Call startNewEpoch before setting epoch structure.');
+    }
+
+    this.logger.info(
+      `Setting epoch ${this.provingState.epochNumber} structure with ${totalNumCheckpoints} checkpoints.`,
+    );
+    this.provingState.setStructure(totalNumCheckpoints, finalBlobBatchingChallenges);
+
+    // Update blob batching challenges on all existing checkpoints.
+    for (let i = 0; i < totalNumCheckpoints; i++) {
+      const checkpoint = this.provingState.getCheckpointProvingState(i);
+      if (checkpoint) {
+        checkpoint.setFinalBlobBatchingChallenges(finalBlobBatchingChallenges);
+      }
+    }
+
+    // Re-trigger accumulation for all completed checkpoints.
+    await this.provingState.accumulateCheckpointOutHashes();
+    await this.provingState.setBlobAccumulators();
+  }
+
   /**
    * Starts a new checkpoint.
    * @param checkpointIndex - The index of the checkpoint in the epoch.