Skip to content

refactor: add getCheckpointsDataForEpoch and enrich CheckpointData #20467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
PhilWindle merged 1 commit into from
Feb 16, 2026
Merged

refactor: add getCheckpointsDataForEpoch and enrich CheckpointData #20467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 45 additions & 99 deletions yarn-project/archiver/src/modules/data_source_base.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,12 @@
import { range } from '@aztec/foundation/array';
import { BlockNumber, CheckpointNumber, type EpochNumber, type SlotNumber } from '@aztec/foundation/branded-types';
import type { Fr } from '@aztec/foundation/curves/bn254';
import type { EthAddress } from '@aztec/foundation/eth-address';
import { isDefined } from '@aztec/foundation/types';
import type { FunctionSelector } from '@aztec/stdlib/abi';
import type { AztecAddress } from '@aztec/stdlib/aztec-address';
import {
type BlockData,
type BlockHash,
CheckpointedL2Block,
CommitteeAttestation,
L2Block,
type L2Tips,
} from '@aztec/stdlib/block';
import { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
import { type BlockData, type BlockHash, CheckpointedL2Block, L2Block, type L2Tips } from '@aztec/stdlib/block';
import { Checkpoint, type CheckpointData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
import type { ContractClassPublic, ContractDataSource, ContractInstanceWithAddress } from '@aztec/stdlib/contract';
import { type L1RollupConstants, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
import type { GetContractClassLogsResponse, GetPublicLogsResponse } from '@aztec/stdlib/interfaces/client';
Expand All @@ -24,7 +18,6 @@ import type { BlockHeader, IndexedTxEffect, TxHash, TxReceipt } from '@aztec/std
import type { UInt64 } from '@aztec/stdlib/types';

import type { ArchiverDataSource } from '../interfaces.js';
import type { CheckpointData } from '../store/block_store.js';
import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
import type { ValidateCheckpointResult } from './validation.js';

Expand Down Expand Up @@ -121,7 +114,7 @@ export abstract class ArchiverDataSourceBase
if (!checkpointData) {
return undefined;
}
return BlockNumber(checkpointData.startBlock + checkpointData.numBlocks - 1);
return BlockNumber(checkpointData.startBlock + checkpointData.blockCount - 1);
}

public getCheckpointedBlocks(from: BlockNumber, limit: number): Promise<CheckpointedL2Block[]> {
Expand Down Expand Up @@ -238,113 +231,66 @@ export abstract class ArchiverDataSourceBase

public async getCheckpoints(checkpointNumber: CheckpointNumber, limit: number): Promise<PublishedCheckpoint[]> {
const checkpoints = await this.store.getRangeOfCheckpoints(checkpointNumber, limit);
const blocks = (
await Promise.all(checkpoints.map(ch => this.store.getBlocksForCheckpoint(ch.checkpointNumber)))
).filter(isDefined);

const fullCheckpoints: PublishedCheckpoint[] = [];
for (let i = 0; i < checkpoints.length; i++) {
const blocksForCheckpoint = blocks[i];
const checkpoint = checkpoints[i];
const fullCheckpoint = new Checkpoint(
checkpoint.archive,
checkpoint.header,
blocksForCheckpoint,
checkpoint.checkpointNumber,
);
const publishedCheckpoint = new PublishedCheckpoint(
fullCheckpoint,
checkpoint.l1,
checkpoint.attestations.map(x => CommitteeAttestation.fromBuffer(x)),
);
fullCheckpoints.push(publishedCheckpoint);
return Promise.all(checkpoints.map(ch => this.getPublishedCheckpointFromCheckpointData(ch)));
}

private async getPublishedCheckpointFromCheckpointData(checkpoint: CheckpointData): Promise<PublishedCheckpoint> {
const blocksForCheckpoint = await this.store.getBlocksForCheckpoint(checkpoint.checkpointNumber);
if (!blocksForCheckpoint) {
throw new Error(`Blocks for checkpoint ${checkpoint.checkpointNumber} not found`);
}
return fullCheckpoints;
const fullCheckpoint = new Checkpoint(
checkpoint.archive,
checkpoint.header,
blocksForCheckpoint,
checkpoint.checkpointNumber,
);
return new PublishedCheckpoint(fullCheckpoint, checkpoint.l1, checkpoint.attestations);
}

public getBlocksForSlot(slotNumber: SlotNumber): Promise<L2Block[]> {
return this.store.getBlocksForSlot(slotNumber);
}

public async getCheckpointedBlocksForEpoch(epochNumber: EpochNumber): Promise<CheckpointedL2Block[]> {
if (!this.l1Constants) {
throw new Error('L1 constants not set');
}

const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1Constants);
const blocks: CheckpointedL2Block[] = [];

// Walk the list of checkpoints backwards and filter by slots matching the requested epoch.
// We'll typically ask for checkpoints for a very recent epoch, so we shouldn't need an index here.
let checkpoint = await this.store.getCheckpointData(await this.store.getSynchedCheckpointNumber());
const slot = (b: CheckpointData) => b.header.slotNumber;
while (checkpoint && slot(checkpoint) >= start) {
if (slot(checkpoint) <= end) {
// push the blocks on backwards
const endBlock = checkpoint.startBlock + checkpoint.numBlocks - 1;
for (let i = endBlock; i >= checkpoint.startBlock; i--) {
const checkpointedBlock = await this.getCheckpointedBlock(BlockNumber(i));
if (checkpointedBlock) {
blocks.push(checkpointedBlock);
}
}
}
checkpoint = await this.store.getCheckpointData(CheckpointNumber(checkpoint.checkpointNumber - 1));
}

return blocks.reverse();
const checkpointsData = await this.getCheckpointsDataForEpoch(epochNumber);
const blocks = await Promise.all(
checkpointsData.flatMap(checkpoint =>
range(checkpoint.blockCount, checkpoint.startBlock).map(blockNumber =>
this.getCheckpointedBlock(BlockNumber(blockNumber)),
),
),
);
return blocks.filter(isDefined);
}

public async getCheckpointedBlockHeadersForEpoch(epochNumber: EpochNumber): Promise<BlockHeader[]> {
if (!this.l1Constants) {
throw new Error('L1 constants not set');
}

const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1Constants);
const blocks: BlockHeader[] = [];

// Walk the list of checkpoints backwards and filter by slots matching the requested epoch.
// We'll typically ask for checkpoints for a very recent epoch, so we shouldn't need an index here.
let checkpoint = await this.store.getCheckpointData(await this.store.getSynchedCheckpointNumber());
const slot = (b: CheckpointData) => b.header.slotNumber;
while (checkpoint && slot(checkpoint) >= start) {
if (slot(checkpoint) <= end) {
// push the blocks on backwards
const endBlock = checkpoint.startBlock + checkpoint.numBlocks - 1;
for (let i = endBlock; i >= checkpoint.startBlock; i--) {
const block = await this.getBlockHeader(BlockNumber(i));
if (block) {
blocks.push(block);
}
}
}
checkpoint = await this.store.getCheckpointData(CheckpointNumber(checkpoint.checkpointNumber - 1));
}
return blocks.reverse();
const checkpointsData = await this.getCheckpointsDataForEpoch(epochNumber);
const blocks = await Promise.all(
checkpointsData.flatMap(checkpoint =>
range(checkpoint.blockCount, checkpoint.startBlock).map(blockNumber =>
this.getBlockHeader(BlockNumber(blockNumber)),
),
),
);
return blocks.filter(isDefined);
}

public async getCheckpointsForEpoch(epochNumber: EpochNumber): Promise<Checkpoint[]> {
const checkpointsData = await this.getCheckpointsDataForEpoch(epochNumber);
return Promise.all(
checkpointsData.map(data => this.getPublishedCheckpointFromCheckpointData(data).then(p => p.checkpoint)),
);
}

/** Returns checkpoint data for all checkpoints whose slot falls within the given epoch. */
public getCheckpointsDataForEpoch(epochNumber: EpochNumber): Promise<CheckpointData[]> {
if (!this.l1Constants) {
throw new Error('L1 constants not set');
}

const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1Constants);
const checkpoints: Checkpoint[] = [];

// Walk the list of checkpoints backwards and filter by slots matching the requested epoch.
// We'll typically ask for checkpoints for a very recent epoch, so we shouldn't need an index here.
let checkpointData = await this.store.getCheckpointData(await this.store.getSynchedCheckpointNumber());
const slot = (b: CheckpointData) => b.header.slotNumber;
while (checkpointData && slot(checkpointData) >= start) {
if (slot(checkpointData) <= end) {
// push the checkpoints on backwards
const [checkpoint] = await this.getCheckpoints(checkpointData.checkpointNumber, 1);
checkpoints.push(checkpoint.checkpoint);
}
checkpointData = await this.store.getCheckpointData(CheckpointNumber(checkpointData.checkpointNumber - 1));
}

return checkpoints.reverse();
return this.store.getCheckpointDataForSlotRange(start, end);
}

public async getBlock(number: BlockNumber): Promise<L2Block | undefined> {
Expand Down
5 changes: 2 additions & 3 deletions yarn-project/archiver/src/modules/instrumentation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { createLogger } from '@aztec/foundation/log';
import type { L2Block } from '@aztec/stdlib/block';
import type { CheckpointData } from '@aztec/stdlib/checkpoint';
import {
Attributes,
type Gauge,
Expand All @@ -13,8 +14,6 @@ import {
createUpDownCounterWithDefault,
} from '@aztec/telemetry-client';

import type { CheckpointData } from '../store/block_store.js';

export class ArchiverInstrumentation {
public readonly tracer: Tracer;

Expand Down Expand Up @@ -134,7 +133,7 @@ export class ArchiverInstrumentation {
}

public updateLastProvenCheckpoint(checkpoint: CheckpointData) {
const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.numBlocks - 1;
const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.blockCount - 1;
this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' });
this.checkpointHeight.record(checkpoint.checkpointNumber, { [Attributes.STATUS]: 'proven' });
}
Expand Down
2 changes: 1 addition & 1 deletion yarn-project/archiver/src/modules/l1_synchronizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,7 +590,7 @@ export class ArchiverL1Synchronizer implements Traceable {
const provenEpochNumber: EpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants);
const lastBlockNumberInCheckpoint =
localCheckpointForDestinationProvenCheckpointNumber.startBlock +
localCheckpointForDestinationProvenCheckpointNumber.numBlocks -
localCheckpointForDestinationProvenCheckpointNumber.blockCount -
1;

this.events.emit(L2BlockSourceEvents.L2BlockProven, {
Expand Down
67 changes: 43 additions & 24 deletions yarn-project/archiver/src/store/block_store.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
deserializeValidateCheckpointResult,
serializeValidateCheckpointResult,
} from '@aztec/stdlib/block';
import { L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
import { type CheckpointData, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
import { CheckpointHeader } from '@aztec/stdlib/rollup';
import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees';
Expand Down Expand Up @@ -62,23 +62,14 @@ type BlockStorage = {
type CheckpointStorage = {
header: Buffer;
archive: Buffer;
checkpointOutHash: Buffer;
checkpointNumber: number;
startBlock: number;
numBlocks: number;
blockCount: number;
l1: Buffer;
attestations: Buffer[];
};

export type CheckpointData = {
checkpointNumber: CheckpointNumber;
header: CheckpointHeader;
archive: AppendOnlyTreeSnapshot;
startBlock: number;
numBlocks: number;
l1: L1PublishedData;
attestations: Buffer[];
};

export type RemoveCheckpointsResult = { blocksRemoved: L2Block[] | undefined };

/**
Expand All @@ -91,6 +82,9 @@ export class BlockStore {
/** Map checkpoint number to checkpoint data */
#checkpoints: AztecAsyncMap<number, CheckpointStorage>;

/** Map slot number to checkpoint number, for looking up checkpoints by slot range. */
#slotToCheckpoint: AztecAsyncMap<number, number>;

/** Map block hash to list of tx hashes */
#blockTxs: AztecAsyncMap<string, Buffer>;

Expand Down Expand Up @@ -131,6 +125,7 @@ export class BlockStore {
this.#lastProvenCheckpoint = db.openSingleton('archiver_last_proven_l2_checkpoint');
this.#pendingChainValidationStatus = db.openSingleton('archiver_pending_chain_validation_status');
this.#checkpoints = db.openMap('archiver_checkpoints');
this.#slotToCheckpoint = db.openMap('archiver_slot_to_checkpoint');
}

/**
Expand Down Expand Up @@ -274,7 +269,7 @@ export class BlockStore {

// If we have a previous checkpoint then we need to get the previous block number
if (previousCheckpointData !== undefined) {
previousBlockNumber = BlockNumber(previousCheckpointData.startBlock + previousCheckpointData.numBlocks - 1);
previousBlockNumber = BlockNumber(previousCheckpointData.startBlock + previousCheckpointData.blockCount - 1);
previousBlock = await this.getBlock(previousBlockNumber);
if (previousBlock === undefined) {
// We should be able to get the required previous block
Expand Down Expand Up @@ -338,12 +333,16 @@ export class BlockStore {
await this.#checkpoints.set(checkpoint.checkpoint.number, {
header: checkpoint.checkpoint.header.toBuffer(),
archive: checkpoint.checkpoint.archive.toBuffer(),
checkpointOutHash: checkpoint.checkpoint.getCheckpointOutHash().toBuffer(),
l1: checkpoint.l1.toBuffer(),
attestations: checkpoint.attestations.map(attestation => attestation.toBuffer()),
checkpointNumber: checkpoint.checkpoint.number,
startBlock: checkpoint.checkpoint.blocks[0].number,
numBlocks: checkpoint.checkpoint.blocks.length,
blockCount: checkpoint.checkpoint.blocks.length,
});

// Update slot-to-checkpoint index
await this.#slotToCheckpoint.set(checkpoint.checkpoint.header.slotNumber, checkpoint.checkpoint.number);
}

await this.#lastSynchedL1Block.set(checkpoints[checkpoints.length - 1].l1.blockNumber);
Expand Down Expand Up @@ -426,14 +425,19 @@ export class BlockStore {
if (!targetCheckpoint) {
throw new Error(`Target checkpoint ${checkpointNumber} not found in store`);
}
lastBlockToKeep = BlockNumber(targetCheckpoint.startBlock + targetCheckpoint.numBlocks - 1);
lastBlockToKeep = BlockNumber(targetCheckpoint.startBlock + targetCheckpoint.blockCount - 1);
}

// Remove all blocks after lastBlockToKeep (both checkpointed and uncheckpointed)
const blocksRemoved = await this.removeBlocksAfter(lastBlockToKeep);

// Remove all checkpoints after the target
for (let c = latestCheckpointNumber; c > checkpointNumber; c = CheckpointNumber(c - 1)) {
const checkpointStorage = await this.#checkpoints.getAsync(c);
if (checkpointStorage) {
const slotNumber = CheckpointHeader.fromBuffer(checkpointStorage.header).slotNumber;
await this.#slotToCheckpoint.delete(slotNumber);
}
await this.#checkpoints.delete(c);
this.#log.debug(`Removed checkpoint ${c}`);
}
Expand Down Expand Up @@ -462,17 +466,32 @@ export class BlockStore {
return checkpoints;
}

private checkpointDataFromCheckpointStorage(checkpointStorage: CheckpointStorage) {
const data: CheckpointData = {
/** Returns checkpoint data for all checkpoints whose slot falls within the given range (inclusive). */
async getCheckpointDataForSlotRange(startSlot: SlotNumber, endSlot: SlotNumber): Promise<CheckpointData[]> {
const result: CheckpointData[] = [];
for await (const [, checkpointNumber] of this.#slotToCheckpoint.entriesAsync({
start: startSlot,
end: endSlot + 1,
})) {
const checkpointStorage = await this.#checkpoints.getAsync(checkpointNumber);
if (checkpointStorage) {
result.push(this.checkpointDataFromCheckpointStorage(checkpointStorage));
}
}
return result;
}

private checkpointDataFromCheckpointStorage(checkpointStorage: CheckpointStorage): CheckpointData {
return {
header: CheckpointHeader.fromBuffer(checkpointStorage.header),
archive: AppendOnlyTreeSnapshot.fromBuffer(checkpointStorage.archive),
checkpointOutHash: Fr.fromBuffer(checkpointStorage.checkpointOutHash),
checkpointNumber: CheckpointNumber(checkpointStorage.checkpointNumber),
startBlock: checkpointStorage.startBlock,
numBlocks: checkpointStorage.numBlocks,
startBlock: BlockNumber(checkpointStorage.startBlock),
blockCount: checkpointStorage.blockCount,
l1: L1PublishedData.fromBuffer(checkpointStorage.l1),
attestations: checkpointStorage.attestations,
attestations: checkpointStorage.attestations.map(buf => CommitteeAttestation.fromBuffer(buf)),
};
return data;
}

async getBlocksForCheckpoint(checkpointNumber: CheckpointNumber): Promise<L2Block[] | undefined> {
Expand All @@ -484,7 +503,7 @@ export class BlockStore {
const blocksForCheckpoint = await toArray(
this.#blocks.entriesAsync({
start: checkpoint.startBlock,
end: checkpoint.startBlock + checkpoint.numBlocks,
end: checkpoint.startBlock + checkpoint.blockCount,
}),
);

Expand Down Expand Up @@ -557,7 +576,7 @@ export class BlockStore {
if (!checkpointStorage) {
throw new CheckpointNotFoundError(provenCheckpointNumber);
} else {
return BlockNumber(checkpointStorage.startBlock + checkpointStorage.numBlocks - 1);
return BlockNumber(checkpointStorage.startBlock + checkpointStorage.blockCount - 1);
}
}

Expand Down Expand Up @@ -922,7 +941,7 @@ export class BlockStore {
if (!checkpoint) {
return BlockNumber(INITIAL_L2_BLOCK_NUM - 1);
}
return BlockNumber(checkpoint.startBlock + checkpoint.numBlocks - 1);
return BlockNumber(checkpoint.startBlock + checkpoint.blockCount - 1);
}

async getLatestL2BlockNumber(): Promise<BlockNumber> {
Expand Down
Loading
Loading