refactor(archiver): refactor archiver modules and test suite

yarn-project/archiver/package.json

@@ -4,10 +4,8 @@
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
-    "./data-retrieval": "./dest/archiver/data_retrieval.js",
-    "./epoch": "./dest/archiver/epoch_helpers.js",
     "./test": "./dest/test/index.js",
-    "./config": "./dest/archiver/config.js"
+    "./config": "./dest/config.js"
   },
   "typedocOptions": {
     "entryPoints": [

yarn-project/archiver/src/archiver/archiver.sync.test.ts → yarn-project/archiver/src/archiver-sync.test.ts

@@ -5,7 +5,7 @@ import type { EpochCache, EpochCommitteeInfo } from '@aztec/epoch-cache';
 import { DefaultL1ContractsConfig } from '@aztec/ethereum/config';
 import { BlockTagTooOldError, type InboxContract, type RollupContract } from '@aztec/ethereum/contracts';
 import type { ViemPublicClient } from '@aztec/ethereum/types';
-import { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
 import { Buffer32 } from '@aztec/foundation/buffer';
 import { sum, times } from '@aztec/foundation/collection';
 import { Secp256k1Signer } from '@aztec/foundation/crypto/secp256k1-signer';
@@ -22,12 +22,14 @@ import { getTelemetryClient } from '@aztec/telemetry-client';
 
 import { jest } from '@jest/globals';
 import assert from 'assert';
+import { EventEmitter } from 'events';
 import { type MockProxy, mock } from 'jest-mock-extended';
 import type { GetBlockReturnType } from 'viem';
 
-import { Archiver } from './archiver.js';
-import type { ArchiverInstrumentation } from './instrumentation.js';
-import { KVArchiverDataStore } from './kv_archiver_store/kv_archiver_store.js';
+import { Archiver, type ArchiverEmitter } from './archiver.js';
+import type { ArchiverInstrumentation } from './modules/instrumentation.js';
+import { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js';
+import { KVArchiverDataStore } from './store/kv_archiver_store.js';
 import { FakeL1State, type FakeL1StateConfig } from './test/fake_l1_state.js';
 
 describe('Archiver Sync', () => {
@@ -49,14 +51,17 @@ describe('Archiver Sync', () => {
   let archiverStore: KVArchiverDataStore;
   let l1Constants: L1RollupConstants & { l1StartBlockHash: Buffer32; genesisArchiveRoot: Fr };
   let archiver: Archiver;
+  let synchronizer: ArchiverL1Synchronizer;
   let logger: Logger;
+  let syncLogger: Logger;
   let now: number;
 
   const GENESIS_ROOT = new Fr(GENESIS_ARCHIVE_ROOT);
   const ETHEREUM_SLOT_DURATION = BigInt(DefaultL1ContractsConfig.ethereumSlotDuration);
 
   beforeEach(async () => {
     logger = createLogger('archiver:sync:test');
+    syncLogger = createLogger('archiver:l1-sync:test');
     now = Math.floor(Date.now() / 1000);
     dateProvider = new TestDateProvider();
 
@@ -109,24 +114,47 @@ describe('Archiver Sync', () => {
     rollupContract = fake.createMockRollupContract(publicClient);
     inboxContract = fake.createMockInboxContract(publicClient);
 
-    archiver = new Archiver(
+    const config = {
+      pollingIntervalMs: 1000,
+      batchSize: 1000,
+      maxAllowedEthClientDriftSeconds: 300,
+      ethereumAllowNoDebugHosts: true,
+    };
+
+    // Create event emitter shared by archiver and synchronizer
+    const events = new EventEmitter() as ArchiverEmitter;
+
+    // Create the L1 synchronizer
+    synchronizer = new ArchiverL1Synchronizer(
+      publicClient,
       publicClient,
-      publicClient, // debugClient same as publicClient for tests
       rollupContract,
       inboxContract,
       contractAddresses,
       archiverStore,
-      {
-        pollingIntervalMs: 1000,
-        batchSize: 1000,
-        maxAllowedEthClientDriftSeconds: 300,
-        ethereumAllowNoDebugHosts: true,
-      },
+      config,
       blobClient,
       epochCache,
       dateProvider,
       instrumentation,
       l1Constants,
+      events,
+      instrumentation.tracer,
+      syncLogger,
+    );
+
+    archiver = new Archiver(
+      publicClient,
+      publicClient,
+      rollupContract,
+      contractAddresses,
+      archiverStore,
+      config,
+      blobClient,
+      instrumentation,
+      l1Constants,
+      synchronizer,
+      events,
     );
   });
 
@@ -217,7 +245,7 @@ describe('Archiver Sync', () => {
     }, 30_000);
 
     it('ignores checkpoint 3 because it has been pruned', async () => {
-      const loggerSpy = jest.spyOn((archiver as any).log, 'warn');
+      const loggerSpy = jest.spyOn(syncLogger, 'warn');
 
       expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(0));
 
@@ -311,7 +339,7 @@ describe('Archiver Sync', () => {
     }, 10_000);
 
     it('skip event search if no changes found', async () => {
-      const loggerSpy = jest.spyOn((archiver as any).log, 'debug');
+      const loggerSpy = jest.spyOn(syncLogger, 'debug');
 
       expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(0));
 
@@ -755,7 +783,7 @@ describe('Archiver Sync', () => {
 
   describe('reorg handling', () => {
     it('handles L2 reorg', async () => {
-      const loggerSpy = jest.spyOn((archiver as any).log, 'debug');
+      const loggerSpy = jest.spyOn(syncLogger, 'debug');
 
       expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(0));
 
@@ -929,4 +957,193 @@ describe('Archiver Sync', () => {
 
     xit('does not attempt to download data for a checkpoint that has been pruned', () => {});
   });
+
+  describe('addBlock and L1 sync interaction', () => {
+    it('blocks added via addBlock become checkpointed when checkpoint syncs from L1', async () => {
+      // First, sync checkpoint 1 from L1 to establish a baseline
+      const { checkpoint: cp1 } = await fake.addCheckpoint(CheckpointNumber(1), {
+        l1BlockNumber: 70n,
+        messagesL1BlockNumber: 60n,
+        numL1ToL2Messages: 3,
+      });
+
+      fake.setL1BlockNumber(100n);
+      await archiver.start(false);
+      await retryUntil(
+        () => archiver.getSynchedCheckpointNumber().then(n => n === CheckpointNumber(1)),
+        'sync',
+        10,
+        0.1,
+      );
+
+      expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(1));
+      const lastBlockInCheckpoint1 = cp1.blocks[cp1.blocks.length - 1].number;
+
+      // Verify L2Tips after syncing checkpoint 1: proposed and checkpointed should both be at checkpoint 1
+      const tipsAfterCheckpoint1 = await archiver.getL2Tips();
+      expect(tipsAfterCheckpoint1.proposed.number).toEqual(lastBlockInCheckpoint1);
+      expect(tipsAfterCheckpoint1.checkpointed.block.number).toEqual(lastBlockInCheckpoint1);
+      expect(tipsAfterCheckpoint1.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1));
+
+      // Create checkpoint 2 on L1 at a future block (not yet visible to archiver)
+      const { checkpoint: cp2 } = await fake.addCheckpoint(CheckpointNumber(2), {
+        l1BlockNumber: 5000n, // Far in the future
+        messagesL1BlockNumber: 4990n,
+        numL1ToL2Messages: 3,
+      });
+
+      // Now add blocks from checkpoint 2 via addBlock (simulating local block production)
+      for (const block of cp2.blocks) {
+        await archiver.addBlock(block);
+      }
+
+      // Verify blocks are retrievable but not yet checkpointed
+      const lastBlockInCheckpoint2 = cp2.blocks[cp2.blocks.length - 1].number;
+      expect(await archiver.getBlockNumber()).toEqual(lastBlockInCheckpoint2);
+      expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(1));
+
+      // Verify L2Tips after adding blocks: proposed advances but checkpointed stays at checkpoint 1
+      const tipsAfterAddBlock = await archiver.getL2Tips();
+      expect(tipsAfterAddBlock.proposed.number).toEqual(lastBlockInCheckpoint2);
+      expect(tipsAfterAddBlock.checkpointed.block.number).toEqual(lastBlockInCheckpoint1);
+      expect(tipsAfterAddBlock.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1));
+
+      // getCheckpointedBlock should return undefined for the new blocks since checkpoint 2 hasn't synced
+      const firstNewBlockNumber = BlockNumber(lastBlockInCheckpoint1 + 1);
+      const uncheckpointedBlock = await archiver.getCheckpointedBlock(firstNewBlockNumber);
+      expect(uncheckpointedBlock).toBeUndefined();
+
+      // But getL2BlockNew should work (it retrieves both checkpointed and uncheckpointed blocks)
+      const block = await archiver.getL2BlockNew(firstNewBlockNumber);
+      expect(block).toBeDefined();
+
+      // Now advance L1 so checkpoint 2 becomes visible
+      fake.setL1BlockNumber(5010n);
+
+      await retryUntil(
+        () => archiver.getSynchedCheckpointNumber().then(n => n === CheckpointNumber(2)),
+        'sync',
+        10,
+        0.1,
+      );
+
+      // Now the blocks should be checkpointed
+      expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(2));
+
+      // Verify L2Tips after syncing checkpoint 2: proposed and checkpointed should both be at checkpoint 2
+      const tipsAfterCheckpoint2 = await archiver.getL2Tips();
+      expect(tipsAfterCheckpoint2.proposed.number).toEqual(lastBlockInCheckpoint2);
+      expect(tipsAfterCheckpoint2.checkpointed.block.number).toEqual(lastBlockInCheckpoint2);
+      expect(tipsAfterCheckpoint2.checkpointed.checkpoint.number).toEqual(CheckpointNumber(2));
+
+      // getCheckpointedBlock should now work for the new blocks
+      const checkpointedBlock = await archiver.getCheckpointedBlock(firstNewBlockNumber);
+      expect(checkpointedBlock).toBeDefined();
+      expect(checkpointedBlock!.checkpointNumber).toEqual(2);
+    }, 10_000);
+
+    it('blocks added via checkpoints can not be added via addBlocks', async () => {
+      // First, sync checkpoint 1 from L1 to establish a baseline
+      const { checkpoint: cp1 } = await fake.addCheckpoint(CheckpointNumber(1), {
+        l1BlockNumber: 70n,
+        messagesL1BlockNumber: 60n,
+        numL1ToL2Messages: 3,
+      });
+
+      fake.setL1BlockNumber(100n);
+      await archiver.start(false);
+      await retryUntil(
+        () => archiver.getSynchedCheckpointNumber().then(n => n === CheckpointNumber(1)),
+        'sync',
+        10,
+        0.1,
+      );
+
+      expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(1));
+      const blockAlreadySyncedFromCheckpoint = cp1.blocks[cp1.blocks.length - 1];
+
+      // Now try and add one of the blocks via the addBlocks method. It should throw
+      await expect(archiver.addBlock(blockAlreadySyncedFromCheckpoint)).rejects.toThrow();
+    }, 10_000);
+
+    it('can add more blocks after checkpoint syncs and then sync another checkpoint', async () => {
+      // Sync the first checkpoint normally
+      const { checkpoint: cp1 } = await fake.addCheckpoint(CheckpointNumber(1), {
+        l1BlockNumber: 70n,
+        messagesL1BlockNumber: 60n,
+        numL1ToL2Messages: 3,
+      });
+
+      fake.setL1BlockNumber(100n);
+      await archiver.start(false);
+      await retryUntil(
+        () => archiver.getSynchedCheckpointNumber().then(n => n === CheckpointNumber(1)),
+        'sync',
+        10,
+        0.1,
+      );
+
+      expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(1));
+      const lastBlockInCheckpoint1 = cp1.blocks[cp1.blocks.length - 1].number;
+
+      // Verify L2Tips after syncing checkpoint 1: proposed and checkpointed at checkpoint 1
+      const tipsAfterCheckpoint1 = await archiver.getL2Tips();
+      expect(tipsAfterCheckpoint1.proposed.number).toEqual(lastBlockInCheckpoint1);
+      expect(tipsAfterCheckpoint1.checkpointed.block.number).toEqual(lastBlockInCheckpoint1);
+      expect(tipsAfterCheckpoint1.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1));
+
+      // Create checkpoint 2 on L1 at a future block (not yet visible)
+      const { checkpoint: cp2 } = await fake.addCheckpoint(CheckpointNumber(2), {
+        l1BlockNumber: 5000n, // Far in the future
+        messagesL1BlockNumber: 4990n,
+        numL1ToL2Messages: 3,
+      });
+
+      // Now add more blocks via addBlock (simulating local block production ahead of L1)
+      for (const block of cp2.blocks) {
+        await archiver.addBlock(block);
+      }
+
+      // Verify blocks are retrievable
+      const lastBlockInCheckpoint2 = cp2.blocks[cp2.blocks.length - 1].number;
+      expect(await archiver.getBlockNumber()).toEqual(lastBlockInCheckpoint2);
+
+      // But checkpoint number should still be 1
+      expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(1));
+
+      // Verify L2Tips after adding blocks: proposed advances, checkpointed stays at checkpoint 1
+      const tipsAfterAddBlock = await archiver.getL2Tips();
+      expect(tipsAfterAddBlock.proposed.number).toEqual(lastBlockInCheckpoint2);
+      expect(tipsAfterAddBlock.checkpointed.block.number).toEqual(lastBlockInCheckpoint1);
+      expect(tipsAfterAddBlock.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1));
+
+      // New blocks should not be checkpointed yet
+      const firstNewBlockNumber = BlockNumber(lastBlockInCheckpoint1 + 1);
+      const uncheckpointedBlock = await archiver.getCheckpointedBlock(firstNewBlockNumber);
+      expect(uncheckpointedBlock).toBeUndefined();
+
+      // Now advance L1 so checkpoint 2 becomes visible
+      fake.setL1BlockNumber(5010n);
+
+      await retryUntil(
+        () => archiver.getSynchedCheckpointNumber().then(n => n === CheckpointNumber(2)),
+        'sync',
+        10,
+        0.1,
+      );
+
+      // Now all blocks should be checkpointed
+      expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(2));
+
+      // Verify L2Tips after syncing checkpoint 2: both proposed and checkpointed at checkpoint 2
+      const tipsAfterCheckpoint2 = await archiver.getL2Tips();
+      expect(tipsAfterCheckpoint2.proposed.number).toEqual(lastBlockInCheckpoint2);
+      expect(tipsAfterCheckpoint2.checkpointed.block.number).toEqual(lastBlockInCheckpoint2);
+      expect(tipsAfterCheckpoint2.checkpointed.checkpoint.number).toEqual(CheckpointNumber(2));
+
+      const checkpointedBlock = await archiver.getCheckpointedBlock(firstNewBlockNumber);
+      expect(checkpointedBlock).toBeDefined();
+      expect(checkpointedBlock!.checkpointNumber).toEqual(2);
+    }, 10_000);
+  });
 });